[develop] Add RAVE data processing to fire_emission task for AQM (#628)

* Add RAVE data processing to fire_emission task

* turn off save options by default

* remove typo

* add file check to aqm_lbcs task

* update hash of upp

Externals.cfg

@@ -21,7 +21,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/UPP
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = a644aaa
+hash = 2b2c84a
 local_path = sorc/UPP
 required = True
 
@@ -58,7 +58,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/AQM-utils
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 5ee63d2
+hash = e078c70
 local_path = sorc/AQM-utils
 required = True
 

jobs/JREGIONAL_FIRE_EMISSION

@@ -86,13 +86,21 @@ fi
 #
 #-----------------------------------------------------------------------
 #
+# Set the run directory
+#
+#-----------------------------------------------------------------------
+#
+DATA="${DATA:-${COMIN}${SLASH_ENSMEM_SUBDIR}}"
+mkdir_vrfy -p "${DATA}"
+#
+#-----------------------------------------------------------------------
+#
 # Create the directory where the RAVE fire emission files should be stored
 #
 #-----------------------------------------------------------------------
 #
 export FIRE_EMISSION_STAGING_DIR="${COMINext}/FIRE_EMISSION"
 mkdir_vrfy -p "${FIRE_EMISSION_STAGING_DIR}"
-cd_vrfy ${FIRE_EMISSION_STAGING_DIR}
 #
 #-----------------------------------------------------------------------
 #

modulefiles/tasks/hera/fire_emission.local.lua

@@ -1,2 +1,3 @@
 load("hpss")
-load("miniconda_regional_workflow")
+load("miniconda_online-cmaq")
+load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))

modulefiles/tasks/wcoss2/fire_emission.local.lua

@@ -1 +1,6 @@
 load("python_regional_workflow")
+
+load(pathJoin("udunits", os.getenv("udunits_ver")))
+load(pathJoin("gsl", os.getenv("gsl_ver")))
+load(pathJoin("netcdf", os.getenv("netcdf_ver")))
+load(pathJoin("nco", os.getenv("nco_ver")))

parm/FV3LAM_wflow.xml

@@ -452,10 +452,10 @@ MODULES_RUN_TASK_FP script.
 -->
   <task name="&TN_FIRE_EMISSION;" cycledefs="forecast" maxtries="{{ maxtries_fire_emission }}">
 
-  {%- if do_real_time %}
-    &RSRV_DEFAULT;
-  {%- else %}
+  {%- if do_aqm_save_fire %}
     &RSRV_HPSS;
+  {%- else %}
+    &RSRV_DEFAULT;
   {%- endif %}
     <command>&LOAD_MODULES_RUN_TASK_FP; "&TN_FIRE_EMISSION;" "&JOBSdir;/JREGIONAL_FIRE_EMISSION"</command>
     <nodes>{{ nnodes_fire_emission }}:ppn={{ ppn_fire_emission }}</nodes>
@@ -478,6 +478,7 @@ MODULES_RUN_TASK_FP script.
     <envar><name>cyc</name><value><cyclestr>@H</cyclestr></value></envar>
     <envar><name>subcyc</name><value><cyclestr>@M</cyclestr></value></envar>
     <envar><name>LOGDIR</name><value>&LOGDIR;</value></envar>
+    <envar><name>SLASH_ENSMEM_SUBDIR</name><value><cyclestr>{{ slash_ensmem_subdir }}</cyclestr></value></envar>
   </task>
 {%- endif %}
 {%- if run_task_point_source %}

scripts/exregional_aqm_lbcs.sh

@@ -103,7 +103,7 @@ if [ "${FCST_LEN_HRS}" = "-1" ]; then
   done
 fi
 LBC_SPEC_FCST_HRS=()
-for i_lbc in $(seq ${LBC_SPEC_INTVL_HRS} ${LBC_SPEC_INTVL_HRS} $(( FCST_LEN_HRS+LBC_SPEC_INTVL_HRS )) ); do
+for i_lbc in $(seq ${LBC_SPEC_INTVL_HRS} ${LBC_SPEC_INTVL_HRS} ${FCST_LEN_HRS} ); do
   LBC_SPEC_FCST_HRS+=("$i_lbc")
 done
 
@@ -149,12 +149,23 @@ if [ ${DO_AQM_GEFS_LBCS} = "TRUE" ]; then
   AQM_GEFS_FILE_CYC=${AQM_GEFS_FILE_CYC:-"${CDATE_MOD:8:2}"}
   AQM_GEFS_FILE_CYC=$( printf "%02d" "${AQM_GEFS_FILE_CYC}" )
 
+  AQM_MOFILE_FN="${AQM_GEFS_FILE_PREFIX}.t${AQM_GEFS_FILE_CYC}z.atmf"
   if [ ${DO_REAL_TIME} = "TRUE" ]; then
-    AQM_MOFILE_FN="${COMINgefs}/gefs.${PDY_MOD}/${AQM_GEFS_FILE_CYC}/chem/sfcsig/${AQM_GEFS_FILE_PREFIX}.t${AQM_GEFS_FILE_CYC}z.atmf"
+    AQM_MOFILE_FP="${COMINgefs}/gefs.${PDY_MOD}/${AQM_GEFS_FILE_CYC}/chem/sfcsig/${AQM_MOFILE_FN}"
   else
-    AQM_MOFILE_FN="${AQM_GEFS_DIR}/${PDY}/${AQM_GEFS_FILE_CYC}/${AQM_GEFS_FILE_PREFIX}.t${AQM_GEFS_FILE_CYC}z.atmf"
+    AQM_MOFILE_FP="${AQM_GEFS_DIR}/${PDY}/${AQM_GEFS_FILE_CYC}/${AQM_MOFILE_FN}"
   fi  
 
+  # Check if GEFS aerosol files exist
+  for hr in 0 ${LBC_SPEC_FCST_HRS[@]}; do
+    fhr=$( printf "%03d" "${hr}" )
+    AQM_MOFILE_FHR_FP="${AQM_MOFILE_FP}${fhr}.nemsio"
+    if [ ! -e "${AQM_MOFILE_FHR_FP}" ]; then
+      print_err_msg_exit "The GEFS file (AQM_MOFILE_FHR_FP) for LBCs does not exist:
+      AQM_MOFILE_FHR_FP = \"${AQM_MOFILE_FHR_FP}\""  
+    fi
+  done
+
   GEFS_CYC_DIFF=$( printf "%02d" "$(( RUN_CYC - AQM_GEFS_FILE_CYC ))" )
   NUMTS="$(( FCST_LEN_HRS / LBC_SPEC_INTVL_HRS + 1 ))"
 
@@ -163,7 +174,7 @@ cat > gefs2lbc-nemsio.ini <<EOF
  tstepdiff=${GEFS_CYC_DIFF}
  dtstep=${LBC_SPEC_INTVL_HRS}
  bndname='aothrj','aecj','aorgcj','asoil','numacc','numcor'
- mofile='${AQM_MOFILE_FN}','.nemsio'
+ mofile='${AQM_MOFILE_FP}','.nemsio'
  lbcfile='${INPUT_DATA}/${NET}.${cycle}${dot_ensmem}.gfs_bndy.tile7.f','.nc'
  topofile='${OROG_DIR}/${CRES}_oro_data.tile7.halo4.nc'
 &end

scripts/exregional_fire_emission.sh

@@ -42,43 +42,107 @@ print_info_msg "
 Entering script:  \"${scrfunc_fn}\"
 In directory:     \"${scrfunc_dir}\"
 
-This is the ex-script for the task that copies or fetches fire emission
-data files from disk or HPSS.
+This is the ex-script for the task that fetches fire emission
+data files from disk or generates model-ready RAVE emission file from raw
+data files.
 ========================================================================"
 #
 #-----------------------------------------------------------------------
 #
+# Move to the FIRE EMISSION working directory
+#
+#-----------------------------------------------------------------------
+#
+DATA="${DATA}/tmp_FIRE_EMISSION"
+rm_vrfy -rf $DATA
+mkdir_vrfy -p "$DATA"
+cd_vrfy $DATA
+#
+#-----------------------------------------------------------------------
+#
 # Set up variables for call to retrieve_data.py
 #
 #-----------------------------------------------------------------------
 #
 yyyymmdd=${FIRE_FILE_CDATE:0:8}
 hh=${FIRE_FILE_CDATE:8:2}
+
+CDATE_md1=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - 24 hours" "+%Y%m%d%H" )
+CDATE_mh3=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - 3 hours" "+%Y%m%d%H" )
+yyyymmdd_mh3=${CDATE_mh3:0:8}
+hh_mh3=${CDATE_mh3:8:2}
+CDATE_mh2=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - 2 hours" "+%Y%m%d%H" )
+CDATE_mh1=$( $DATE_UTIL --utc --date "${yyyymmdd} ${hh} UTC - 1 hours" "+%Y%m%d%H" )
+
 #
 #-----------------------------------------------------------------------
 #
-# Retrieve fire files to FIRE_EMISSION_STAGING_DIR
+# Retrieve fire file to FIRE_EMISSION_STAGING_DIR
 #
 #-----------------------------------------------------------------------
 #
 aqm_fire_file_fn="${AQM_FIRE_FILE_PREFIX}_${yyyymmdd}_t${hh}z${AQM_FIRE_FILE_SUFFIX}"
 
-# Check if the file exists in the designated directory
+# Check if the fire file exists in the designated directory
 if [ -e "${AQM_FIRE_DIR}/${yyyymmdd}/${aqm_fire_file_fn}" ]; then
   cp_vrfy "${AQM_FIRE_DIR}/${yyyymmdd}/${aqm_fire_file_fn}" "${FIRE_EMISSION_STAGING_DIR}"
 else
-  # Retrieve files from HPSS
-  arcv_dir="/NCEPDEV/emc-naqfc/2year/Kai.Wang/RAVE_fire/RAVE_NA"
-  arcv_fp="${arcv_dir}/${aqm_fire_file_fn}"
+  # Copy raw data 
+  for ihr in {0..21}; do
+    download_time=$( $DATE_UTIL --utc --date "${yyyymmdd_mh3} ${hh_mh3} UTC - $ihr hours" "+%Y%m%d%H" )
+    FILE_13km="Hourly_Emissions_13km_${download_time}00_${download_time}00.nc"
+    if [ -e "${AQM_FIRE_DIR}/RAVE_raw_new/${FILE_13km}" ]; then
+      ln_vrfy -sf "${AQM_FIRE_DIR}/RAVE_raw_new/Hourly_Emissions_13km_${download_time}00_${download_time}00.nc" .
+    elif [ -d "${AQM_FIRE_DIR}/${CDATE_md1}" ]; then
+      echo "${FILE_13km} does not exist. Replacing with the file of previous date ..."
+      yyyymmdd_dn=${download_time:0:8}
+      hh_dn=${download_time:8:2}
+      missing_download_time=$( $DATE_UTIL --utc --date "${yyyymmdd_dn} ${hh_dn} UTC - 24 hours" "+%Y%m%d%H" )
+      ln_vrfy -sf "${AQM_FIRE_DIR}/${CDATE_md1}/Hourly_Emissions_13km_${missing_download_time}00_${missing_download_time}00.nc" "Hourly_Emissions_13km_${download_time}00_${download_time}00.nc"
+    else
+      print_err_msg_exit "RAVE raw data files do not exist."
+    fi
+  done  
+
+  ncks -O -h --mk_rec_dmn time Hourly_Emissions_13km_${download_time}00_${download_time}00.nc temp.nc || print_err_msg_exit "\
+Call to NCKS returned with nonzero exit code." 
+
+  mv_vrfy temp.nc Hourly_Emissions_13km_${download_time}00_${download_time}00.nc
+
+  # Extra times
+  cp_vrfy Hourly_Emissions_13km_${CDATE_mh3}00_${CDATE_mh3}00.nc Hourly_Emissions_13km_${CDATE_mh2}00_${CDATE_mh2}00.nc
+  cp_vrfy Hourly_Emissions_13km_${CDATE_mh3}00_${CDATE_mh3}00.nc Hourly_Emissions_13km_${CDATE_mh1}00_${CDATE_mh1}00.nc
+
+  ncrcat -h Hourly_Emissions_13km_*.nc Hourly_Emissions_13km_${yyyymmdd}0000_${yyyymmdd}2300.t${cyc}z.nc || print_err_msg_exit "\
+Call to NCRCAT returned with nonzero exit code."
+
+  input_fire="${DATA}/Hourly_Emissions_13km_${yyyymmdd}0000_${yyyymmdd}2300.t${cyc}z.nc"
+  output_fire="${DATA}/Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_new24.t${cyc}z.nc"
+
+  python3 ${HOMEdir}/sorc/AQM-utils/python_utils/RAVE_remake.allspecies.aqmna13km.g793.py --date "${yyyymmdd}" --cyc "${hh}" --input_fire "${input_fire}" --output_fire "${output_fire}"
+
+  ncks --mk_rec_dmn Time Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_new24.t${cyc}z.nc -o Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_t${cyc}z_h24.nc || print_err_msg_exit "\
+Call to NCKS returned with nonzero exit code."
+
+  ncrcat Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_t${cyc}z_h24.nc Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_t${cyc}z_h24.nc Hourly_Emissions_regrid_NA_13km_${yyyymmdd}_t${cyc}z_h24.nc ${aqm_fire_file_fn} || print_err_msg_exit "\
+Call to NCRCAT returned with nonzero exit code."
+
+  # Copy the final fire emission file to STAGING_DIR 
+  cp_vrfy "${DATA}/${aqm_fire_file_fn}" "${FIRE_EMISSION_STAGING_DIR}"
+
+  # Archive the final fire emission file to disk and HPSS
+  if [ "${DO_AQM_SAVE_FIRE}" = "TRUE" ]; then
+    mkdir -p "${AQM_FIRE_DIR}/${yyyymmdd}"
+    cp_vrfy "${DATA}/${aqm_fire_file_fn}" "${AQM_FIRE_DIR}/${yyyymmdd}"
 
-  hsi_log_fn="log.hsi_get.${yyyymmdd}_${hh}"
-  hsi get ${arcv_fp} >& ${hsi_log_fn} || \
+    hsi_log_fn="log.hsi_put.${yyyymmdd}_${hh}"
+    hsi put ${aqm_fire_file_fn} : ${AQM_FIRE_ARCHV_DIR}/${aqm_fire_file_fn} >& ${hsi_log_fn} || \
   print_err_msg_exit "\
-htar file reading operation (\"hsi get ...\") failed.  Check the log 
-file hsi_log_fn in the staging directory (fire_emission_staging_dir) for 
-details:
-  fire_emission_staging_dir = \"${FIRE_EMISSION_STAGING_DIR}\"
+htar file writing operation (\"hsi put ...\") failed.  Check the log 
+file hsi_log_fn in the DATA directory for details:
+  DATA = \"${DATA}\"
   hsi_log_fn = \"${hsi_log_fn}\""
+  fi
 fi
 #
 #-----------------------------------------------------------------------

ush/config.aqm.community.yaml

@@ -10,8 +10,8 @@ workflow:
   EXPT_SUBDIR: aqm_community_aqmna13
   PREDEF_GRID_NAME: AQM_NA_13km
   CCPP_PHYS_SUITE: FV3_GFS_v16
-  DATE_FIRST_CYCL: '2023011700'
-  DATE_LAST_CYCL: '2023011706'
+  DATE_FIRST_CYCL: '2023021700'
+  DATE_LAST_CYCL: '2023021706'
   INCR_CYCL_FREQ: 6
   FCST_LEN_HRS: -1
   FCST_LEN_CYCL:
@@ -73,6 +73,7 @@ cpl_aqm_parm:
   DO_AQM_CANOPY: false
   DO_AQM_PRODUCT: true
   DO_AQM_SAVE_AIRNOW_HIST: false
+  DO_AQM_SAVE_FIRE: false
   AQM_BIO_FILE: BEIS_RRFScmaq_C775.ncf
   AQM_DUST_FILE_PREFIX: FENGSHA_p8_10km_inputs
   AQM_DUST_FILE_SUFFIX: .nc  

ush/config.aqm.nco.realtime.yaml

@@ -10,8 +10,8 @@ workflow:
   EXPT_SUBDIR: aqm_nco_aqmna13km
   PREDEF_GRID_NAME: AQM_NA_13km
   CCPP_PHYS_SUITE: FV3_GFS_v16
-  DATE_FIRST_CYCL: '2023013000'
-  DATE_LAST_CYCL: '2023013018'
+  DATE_FIRST_CYCL: '2023021700'
+  DATE_LAST_CYCL: '2023021718'
   INCR_CYCL_FREQ: 6
   FCST_LEN_HRS: -1
   FCST_LEN_CYCL:
@@ -89,7 +89,8 @@ cpl_aqm_parm:
   DO_AQM_DUST: true
   DO_AQM_CANOPY: false
   DO_AQM_PRODUCT: true
-  DO_AQM_SAVE_AIRNOW_HIST: true
+  DO_AQM_SAVE_AIRNOW_HIST: false
+  DO_AQM_SAVE_FIRE: false
   AQM_BIO_FILE: BEIS_RRFScmaq_C775.ncf
   AQM_DUST_FILE_PREFIX: FENGSHA_p8_10km_inputs
   AQM_DUST_FILE_SUFFIX: .nc

ush/config_defaults.yaml

@@ -2655,6 +2655,9 @@ cpl_aqm_parm:
   # DO_AQM_SAVE_AIRNOW_HIST:
   # Save bias-correction airnow training data
   #
+  # DO_AQM_SAVE_FIRE:
+  # Archive fire emission file to HPSS
+  #
   # AQM_CONFIG_DIR:
   # Configuration directory for AQM
   # 
@@ -2691,6 +2694,9 @@ cpl_aqm_parm:
   # AQM_FIRE_FILE_SUFFIX:
   # Suffix and extension of AQM FIRE file
   #
+  # AQM_FIRE_ARCHV_DIR:
+  # Path to the archive directory for RAVE emission files on HPSS
+  #
   # AQM_RC_FIRE_FREQUENCY:
   # Fire frequency in aqm.rc
   #
@@ -2753,7 +2759,8 @@ cpl_aqm_parm:
   DO_AQM_PRODUCT: true
   DO_AQM_CHEM_LBCS: true
   DO_AQM_GEFS_LBCS: false
-  DO_AQM_SAVE_AIRNOW_HIST: true
+  DO_AQM_SAVE_AIRNOW_HIST: false
+  DO_AQM_SAVE_FIRE: false
 
   AQM_CONFIG_DIR: ""
   AQM_BIO_DIR: ""
@@ -2771,6 +2778,7 @@ cpl_aqm_parm:
   AQM_FIRE_FILE_PREFIX: "GBBEPx_C401GRID.emissions_v003"
   AQM_FIRE_FILE_SUFFIX: ".nc"
   AQM_FIRE_FILE_OFFSET_HRS: 0
+  AQM_FIRE_ARCHV_DIR: "/path/to/archive/dir/for/RAVE/on/HPSS"
 
   AQM_RC_FIRE_FREQUENCY: "static"
   AQM_RC_PRODUCT_FN: "aqm.prod.nc"

ush/machine/hera.yaml

@@ -43,7 +43,7 @@ cpl_aqm_parm:
   AQM_BIO_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/bio
   AQM_DUST_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/FENGSHA
   AQM_CANOPY_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/canopy
-  AQM_FIRE_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/emissions/GSCE/RAVE.in.793/RAVE_RT
+  AQM_FIRE_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/RAVE_fire
   AQM_LBCS_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/LBCS/AQM_NA13km_AM4_v1
   AQM_GEFS_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/GEFS_DATA
   NEXUS_INPUT_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/emissions/nexus

ush/machine/wcoss2.yaml

@@ -43,7 +43,7 @@ cpl_aqm_parm:
   AQM_BIO_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/aqm/bio
   AQM_DUST_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/FENGSHA
   AQM_CANOPY_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/canopy
-  AQM_FIRE_DIR: /lfs/h2/emc/physics/noscrub/jianping.huang/data/RRFS_CMAQ/emissions/GSCE/RAVE.in.C793/RAVE_RT
+  AQM_FIRE_DIR: /lfs/h2/emc/aqmtemp/RAVE_NA
   AQM_LBCS_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/LBCS/AQM_NA13km_AM4_v1
   AQM_GEFS_DIR: /lfs/h2/emc/lam/noscrub/RRFS_CMAQ/GEFS_DATA
   AQM_AIRNOW_HIST_DIR: /lfs/h2/emc/physics/noscrub/jianping.huang/Bias_correction/aqmv7.0