From 14ae595af1b03471287d322596d35c0665336e95 Mon Sep 17 00:00:00 2001 From: shoyokota <103961291+shoyokota@users.noreply.github.com> Date: Mon, 3 Apr 2023 21:27:33 +0900 Subject: [PATCH] GitHub Issue NOAA-EMC/GSI#539 Add options to tune cross-scale/variable/time covariances in EnVar (#542) This PR modifies options (i_ensloccov4{scl,var,tim} -> r_ensloccov4{scl,var,tim}) to tune cross-scale/variable/time covariances in EnVar (https://github.com/NOAA-EMC/GSI/issues/539). Regression tests for global 3dvar/4denvar/4dvar are not completed yet, but for other tests, issues are not found except for "failed the scalability test" and "exceeded maximum allowable hardware memory limit" on Orion. Fixes #539 Co-authored-by: Sho Yokota --- src/gsi/apply_scaledepwgts.f90 | 6 ++--- src/gsi/gsimod.F90 | 31 +++++++++++++---------- src/gsi/hybrid_ensemble_isotropic.F90 | 35 ++++++-------------------- src/gsi/hybrid_ensemble_parameters.f90 | 35 ++++++++++++++++---------- 4 files changed, 50 insertions(+), 57 deletions(-) diff --git a/src/gsi/apply_scaledepwgts.f90 b/src/gsi/apply_scaledepwgts.f90 index 62c455e011..8b93cf0b57 100644 --- a/src/gsi/apply_scaledepwgts.f90 +++ b/src/gsi/apply_scaledepwgts.f90 @@ -47,7 +47,7 @@ subroutine init_mult_spc_wgts(jcap_in) use hybrid_ensemble_parameters,only: use_localization_grid use gridmod,only: use_sp_eqspace use general_specmod, only: general_init_spec_vars - use constants, only: zero,half,one,two,three,rearth,pi + use constants, only: zero,half,one,two,three,rearth,pi,tiny_r_kind use constants, only: rad2deg use mpimod, only: mype use general_sub2grid_mod, only: general_sub2grid_create_info @@ -55,7 +55,7 @@ subroutine init_mult_spc_wgts(jcap_in) use general_sub2grid_mod, only: sub2grid_info use gsi_io, only: verbose use hybrid_ensemble_parameters, only: nsclgrp - use hybrid_ensemble_parameters, only: spc_multwgt,spcwgt_params,i_ensloccov4scl + use hybrid_ensemble_parameters, only: spc_multwgt,spcwgt_params,r_ensloccov4scl implicit none integer(i_kind),intent(in ) :: jcap_in @@ -68,7 +68,7 @@ subroutine init_mult_spc_wgts(jcap_in) integer(i_kind) :: l_sum_spc_weights ! Spectral scale decomposition is differernt between SDL-cross and SDL-nocross - if( i_ensloccov4scl == 1 )then + if( r_ensloccov4scl < tiny_r_kind )then l_sum_spc_weights = 1 else l_sum_spc_weights = 0 diff --git a/src/gsi/gsimod.F90 b/src/gsi/gsimod.F90 index a86c20880c..c4317dbd2c 100644 --- a/src/gsi/gsimod.F90 +++ b/src/gsi/gsimod.F90 @@ -151,7 +151,7 @@ module gsimod readin_beta,use_localization_grid,use_gfs_ens,q_hyb_ens,i_en_perts_io, & l_ens_in_diff_time,ensemble_path,ens_fast_read,sst_staticB,limqens, & ntotensgrp,nsclgrp,naensgrp,ngvarloc,ntlevs_ens,naensloc, & - i_ensloccov4tim,i_ensloccov4var,i_ensloccov4scl,l_timloc_opt,& + r_ensloccov4tim,r_ensloccov4var,r_ensloccov4scl,l_timloc_opt,& vdl_scale,vloc_varlist,& global_spectral_filter_sd,assign_vdl_nml,parallelization_over_ensmembers use hybrid_ensemble_parameters,only : l_both_fv3sar_gfs_ens,n_ens_gfs,n_ens_fv3sar,weight_ens_gfs,weight_ens_fv3sar @@ -1368,23 +1368,28 @@ module gsimod ! l_timloc_opt - if true, then turn on time-dependent localization ! ngvarloc - number of variable-dependent localization lengths ! naensloc - total number of spatial localization lengths and scale separation lengths (should be naensgrp+nsclgrp-1) -! i_ensloccov4tim - flag of cross-temporal localization -! =0: cross-temporal covariance is retained -! =1: cross-temporal covariance is zero -! i_ensloccov4var - flag of cross-variable localization -! =0: cross-variable covariance is retained -! =1: cross-variable covariance is zero -! i_ensloccov4scl - flag of cross-scale localization -! =0: cross-scale covariance is retained -! =1: cross-scale covariance is zero -! +! r_ensloccov4tim - factor multiplying to cross-time covariance +! For example, +! =0.0: cross-time covariance is decreased to zero +! =0.5: cross-time covariance is decreased to half +! =1.0: cross-time covariance is retained +! r_ensloccov4var - factor multiplying to cross-variable covariance +! For example, +! =0.0: cross-variable covariance is decreased to zero +! =0.5: cross-variable covariance is decreased to half +! =1.0: cross-variable covariance is retained +! r_ensloccov4scl - factor multiplying to cross-scale covariance +! For example, +! =0.0: cross-scale covariance is decreased to zero +! =0.5: cross-scale covariance is decreased to half +! =1.0: cross-scale covariance is retained ! global_spectral_filter_sd - if true, use spectral filter function for ! scale decomposition in the global application (Huang et al. 2021) ! assign_vdl_nml - if true, vdl_scale, and vloc_varlist will be used for ! assigning variable-dependent localization upon SDL in gsiparm.anl. ! This method described in (Wang and Wang 2022, JAMES) is ! equivalent to, but different from the method associated -! with the parameter i_ensloccov4var. +! with the parameter r_ensloccov4var. ! vloc_varlist - list of control variables using the same localization length, ! effective only with assign_vdl_nml=.true. For example, ! vloc_varlist(1,:) = 'sf','vp','ps','t', @@ -1415,7 +1420,7 @@ module gsimod grid_ratio_ens, & oz_univ_static,write_ens_sprd,use_localization_grid,use_gfs_ens, & i_en_perts_io,l_ens_in_diff_time,ensemble_path,ens_fast_read,sst_staticB,limqens, & - nsclgrp,l_timloc_opt,ngvarloc,naensloc,i_ensloccov4tim,i_ensloccov4var,i_ensloccov4scl,& + nsclgrp,l_timloc_opt,ngvarloc,naensloc,r_ensloccov4tim,r_ensloccov4var,r_ensloccov4scl,& vdl_scale,vloc_varlist,& global_spectral_filter_sd,assign_vdl_nml,parallelization_over_ensmembers diff --git a/src/gsi/hybrid_ensemble_isotropic.F90 b/src/gsi/hybrid_ensemble_isotropic.F90 index 7e74b3ec27..7b133ffb5a 100644 --- a/src/gsi/hybrid_ensemble_isotropic.F90 +++ b/src/gsi/hybrid_ensemble_isotropic.F90 @@ -5553,7 +5553,7 @@ subroutine setup_ensgrp2aensgrp ! !$$$ end documentation block use constants, only: zero,one - use hybrid_ensemble_parameters, only: l_timloc_opt,i_ensloccov4tim,i_ensloccov4var,i_ensloccov4scl + use hybrid_ensemble_parameters, only: l_timloc_opt,r_ensloccov4tim,r_ensloccov4var,r_ensloccov4scl use hybrid_ensemble_parameters, only: ensloccov4tim,ensloccov4var,ensloccov4scl use hybrid_ensemble_parameters, only: ntotensgrp,naensgrp,ntlevs_ens,nsclgrp,ngvarloc use hybrid_ensemble_parameters, only: ensgrp2aensgrp @@ -5596,33 +5596,12 @@ subroutine setup_ensgrp2aensgrp enddo enddo - if (i_ensloccov4tim==0) then - ensloccov4tim=one - elseif (i_ensloccov4tim==1)then - ensloccov4tim=zero - ensloccov4tim(1)=one - else - write(6,*)'setup_ensgrp2aensgrp: wrong i_ensloccov4tim' - call stop2(666) - endif - if (i_ensloccov4var==0) then - ensloccov4var=one - elseif (i_ensloccov4var==1)then - ensloccov4var=zero - ensloccov4var(1)=one - else - write(6,*)'setup_ensgrp2aensgrp: wrong i_ensloccov4var' - call stop2(666) - endif - if (i_ensloccov4scl==0) then - ensloccov4scl=one - elseif (i_ensloccov4scl==1)then - ensloccov4scl=zero - ensloccov4scl(1)=one - else - write(6,*)'setup_ensgrp2aensgrp: wrong i_ensloccov4scl' - call stop2(666) - endif + ensloccov4tim=r_ensloccov4tim + ensloccov4tim(1)=one + ensloccov4var=r_ensloccov4var + ensloccov4var(1)=one + ensloccov4scl=r_ensloccov4scl + ensloccov4scl(1)=one do itim2=1,ntimloc do itim1=1,ntimloc diff --git a/src/gsi/hybrid_ensemble_parameters.f90 b/src/gsi/hybrid_ensemble_parameters.f90 index d813541c7a..342dead615 100644 --- a/src/gsi/hybrid_ensemble_parameters.f90 +++ b/src/gsi/hybrid_ensemble_parameters.f90 @@ -134,15 +134,21 @@ module hybrid_ensemble_parameters ! l_timloc_opt: if true, then turn on time-dependent localization ! ngvarloc: number of variable-dependent localization lengths ! naensloc: total number of spatial localization lengths and scale separation lengths (should be naensgrp+nsclgrp-1) -! i_ensloccov4tim: flag of cross-temporal localization -! =0: cross-temporal covariance is retained -! =1: cross-temporal covariance is zero -! i_ensloccov4var: flag of cross-variable localization -! =0: cross-variable covariance is retained -! =1: cross-variable covariance is zero -! i_ensloccov4scl: flag of cross-scale localization -! =0: cross-scale covariance is retained -! =1: cross-scale covariance is zero +! r_ensloccov4tim: factor multiplying to cross-time covariance +! For example, +! =0.0: cross-time covariance is decreased to zero +! =0.5: cross-time covariance is decreased to half +! =1.0: cross-time covariance is retained +! r_ensloccov4var: factor multiplying to cross-variable covariance +! For example, +! =0.0: cross-variable covariance is decreased to zero +! =0.5: cross-variable covariance is decreased to half +! =1.0: cross-variable covariance is retained +! r_ensloccov4scl: factor multiplying to cross-scale covariance +! For example, +! =0.0: cross-scale covariance is decreased to zero +! =0.5: cross-scale covariance is decreased to half +! =1.0: cross-scale covariance is retained !===================================================================================================== ! ! @@ -326,7 +332,7 @@ module hybrid_ensemble_parameters public :: ensloccov4tim,ensloccov4var,ensloccov4scl public :: alphacvarsclgrpmat public :: l_timloc_opt - public :: i_ensloccov4tim,i_ensloccov4var,i_ensloccov4scl + public :: r_ensloccov4tim,r_ensloccov4var,r_ensloccov4scl public :: idaen3d,idaen2d public :: ens_fast_read public :: parallelization_over_ensmembers @@ -395,9 +401,9 @@ module hybrid_ensemble_parameters integer(i_kind) :: ntotensgrp=1 integer(i_kind) :: naensloc=1 integer(i_kind) :: ngvarloc=1 - integer(i_kind) :: i_ensloccov4tim=0 - integer(i_kind) :: i_ensloccov4var=0 - integer(i_kind) :: i_ensloccov4scl=0 + real(r_kind) :: r_ensloccov4tim + real(r_kind) :: r_ensloccov4var + real(r_kind) :: r_ensloccov4scl integer(i_kind),allocatable,dimension(:) :: idaen3d,idaen2d real(r_kind),allocatable,dimension(:,:) :: spc_multwgt @@ -501,6 +507,9 @@ subroutine init_hybrid_ensemble_parameters n_ens_fv3sar=0 weight_ens_gfs=one weight_ens_fv3sar=one + r_ensloccov4tim=one + r_ensloccov4var=one + r_ensloccov4scl=one vdl_scale = 0 vloc_varlist = 'aaa' global_spectral_filter_sd=.false.