GitHub Issue #539 Add options to tune cross-scale/variable/time covar…

…iances in EnVar (#542)

This PR modifies options (i_ensloccov4{scl,var,tim} ->
r_ensloccov4{scl,var,tim}) to tune cross-scale/variable/time covariances
in EnVar (#539). Regression tests
for global 3dvar/4denvar/4dvar are not completed yet, but for other
tests, issues are not found except for "failed the scalability test" and
"exceeded maximum allowable hardware memory limit" on Orion.

Fixes #539

Co-authored-by: Sho Yokota <syokota@Orion-login-1.HPC.MsState.Edu>

src/gsi/apply_scaledepwgts.f90

@@ -47,15 +47,15 @@ subroutine init_mult_spc_wgts(jcap_in)
   use hybrid_ensemble_parameters,only: use_localization_grid
   use gridmod,only: use_sp_eqspace
   use general_specmod, only: general_init_spec_vars
-  use constants, only: zero,half,one,two,three,rearth,pi
+  use constants, only: zero,half,one,two,three,rearth,pi,tiny_r_kind
   use constants, only: rad2deg
   use mpimod, only: mype
   use general_sub2grid_mod, only: general_sub2grid_create_info
   use egrid2agrid_mod,only: g_create_egrid2agrid
   use general_sub2grid_mod, only: sub2grid_info
   use gsi_io, only: verbose
   use hybrid_ensemble_parameters, only: nsclgrp
-  use hybrid_ensemble_parameters, only: spc_multwgt,spcwgt_params,i_ensloccov4scl
+  use hybrid_ensemble_parameters, only: spc_multwgt,spcwgt_params,r_ensloccov4scl
   implicit none
 
   integer(i_kind),intent(in   ) :: jcap_in
@@ -68,7 +68,7 @@ subroutine init_mult_spc_wgts(jcap_in)
   integer(i_kind) :: l_sum_spc_weights
 
   ! Spectral scale decomposition is differernt between SDL-cross and SDL-nocross
-  if( i_ensloccov4scl == 1 )then
+  if( r_ensloccov4scl < tiny_r_kind )then
      l_sum_spc_weights = 1
   else
      l_sum_spc_weights = 0

src/gsi/gsimod.F90

@@ -151,7 +151,7 @@ module gsimod
                          readin_beta,use_localization_grid,use_gfs_ens,q_hyb_ens,i_en_perts_io, &
                          l_ens_in_diff_time,ensemble_path,ens_fast_read,sst_staticB,limqens, &
                          ntotensgrp,nsclgrp,naensgrp,ngvarloc,ntlevs_ens,naensloc, &
-                         i_ensloccov4tim,i_ensloccov4var,i_ensloccov4scl,l_timloc_opt,&
+                         r_ensloccov4tim,r_ensloccov4var,r_ensloccov4scl,l_timloc_opt,&
                          vdl_scale,vloc_varlist,&
                          global_spectral_filter_sd,assign_vdl_nml,parallelization_over_ensmembers
   use hybrid_ensemble_parameters,only : l_both_fv3sar_gfs_ens,n_ens_gfs,n_ens_fv3sar,weight_ens_gfs,weight_ens_fv3sar
@@ -1368,23 +1368,28 @@ module gsimod
 !     l_timloc_opt - if true, then turn on time-dependent localization
 !     ngvarloc - number of variable-dependent localization lengths
 !     naensloc - total number of spatial localization lengths and scale separation lengths (should be naensgrp+nsclgrp-1)
-!     i_ensloccov4tim - flag of cross-temporal localization
-!                         =0: cross-temporal covariance is retained
-!                         =1: cross-temporal covariance is zero
-!     i_ensloccov4var - flag of cross-variable localization
-!                         =0: cross-variable covariance is retained
-!                         =1: cross-variable covariance is zero
-!     i_ensloccov4scl - flag of cross-scale localization
-!                         =0: cross-scale covariance is retained
-!                         =1: cross-scale covariance is zero
-!
+!     r_ensloccov4tim - factor multiplying to cross-time covariance
+!                         For example,
+!                         =0.0: cross-time covariance is decreased to zero
+!                         =0.5: cross-time covariance is decreased to half
+!                         =1.0: cross-time covariance is retained
+!     r_ensloccov4var - factor multiplying to cross-variable covariance
+!                         For example,
+!                         =0.0: cross-variable covariance is decreased to zero
+!                         =0.5: cross-variable covariance is decreased to half
+!                         =1.0: cross-variable covariance is retained
+!     r_ensloccov4scl - factor multiplying to cross-scale covariance
+!                         For example,
+!                         =0.0: cross-scale covariance is decreased to zero
+!                         =0.5: cross-scale covariance is decreased to half
+!                         =1.0: cross-scale covariance is retained
 !     global_spectral_filter_sd - if true, use spectral filter function for
 !                                 scale decomposition in the global application (Huang et al. 2021)
 !     assign_vdl_nml - if true, vdl_scale, and vloc_varlist will be used for
 !                      assigning variable-dependent localization upon SDL in gsiparm.anl.
 !                      This method described in (Wang and Wang 2022, JAMES) is
 !                      equivalent to, but different from the method associated
-!                      with the parameter i_ensloccov4var.
+!                      with the parameter r_ensloccov4var.
 !     vloc_varlist - list of control variables using the same localization length,
 !                     effective only with assign_vdl_nml=.true. For example,
 !                     vloc_varlist(1,:) = 'sf','vp','ps','t',
@@ -1415,7 +1420,7 @@ module gsimod
                 grid_ratio_ens, &
                 oz_univ_static,write_ens_sprd,use_localization_grid,use_gfs_ens, &
                 i_en_perts_io,l_ens_in_diff_time,ensemble_path,ens_fast_read,sst_staticB,limqens, &
-                nsclgrp,l_timloc_opt,ngvarloc,naensloc,i_ensloccov4tim,i_ensloccov4var,i_ensloccov4scl,&
+                nsclgrp,l_timloc_opt,ngvarloc,naensloc,r_ensloccov4tim,r_ensloccov4var,r_ensloccov4scl,&
                 vdl_scale,vloc_varlist,&
                 global_spectral_filter_sd,assign_vdl_nml,parallelization_over_ensmembers
 

src/gsi/hybrid_ensemble_isotropic.F90

@@ -5553,7 +5553,7 @@ subroutine setup_ensgrp2aensgrp
 !
 !$$$ end documentation block
   use constants, only: zero,one
-  use hybrid_ensemble_parameters, only: l_timloc_opt,i_ensloccov4tim,i_ensloccov4var,i_ensloccov4scl
+  use hybrid_ensemble_parameters, only: l_timloc_opt,r_ensloccov4tim,r_ensloccov4var,r_ensloccov4scl
   use hybrid_ensemble_parameters, only: ensloccov4tim,ensloccov4var,ensloccov4scl
   use hybrid_ensemble_parameters, only: ntotensgrp,naensgrp,ntlevs_ens,nsclgrp,ngvarloc
   use hybrid_ensemble_parameters, only: ensgrp2aensgrp
@@ -5596,33 +5596,12 @@ subroutine setup_ensgrp2aensgrp
      enddo
   enddo
 
-  if (i_ensloccov4tim==0) then
-     ensloccov4tim=one
-  elseif (i_ensloccov4tim==1)then
-     ensloccov4tim=zero
-     ensloccov4tim(1)=one
-  else
-     write(6,*)'setup_ensgrp2aensgrp: wrong i_ensloccov4tim'
-     call stop2(666)
-  endif
-  if (i_ensloccov4var==0) then
-     ensloccov4var=one
-  elseif (i_ensloccov4var==1)then
-     ensloccov4var=zero
-     ensloccov4var(1)=one
-  else
-     write(6,*)'setup_ensgrp2aensgrp: wrong i_ensloccov4var'
-     call stop2(666)
-  endif
-  if (i_ensloccov4scl==0) then
-     ensloccov4scl=one
-  elseif (i_ensloccov4scl==1)then
-     ensloccov4scl=zero
-     ensloccov4scl(1)=one
-  else
-     write(6,*)'setup_ensgrp2aensgrp: wrong i_ensloccov4scl'
-     call stop2(666)
-  endif
+  ensloccov4tim=r_ensloccov4tim
+  ensloccov4tim(1)=one
+  ensloccov4var=r_ensloccov4var
+  ensloccov4var(1)=one
+  ensloccov4scl=r_ensloccov4scl
+  ensloccov4scl(1)=one
 
   do itim2=1,ntimloc
      do itim1=1,ntimloc

src/gsi/hybrid_ensemble_parameters.f90

@@ -134,15 +134,21 @@ module hybrid_ensemble_parameters
 !      l_timloc_opt:    if true, then turn on time-dependent localization
 !      ngvarloc:        number of variable-dependent localization lengths
 !      naensloc:        total number of spatial localization lengths and scale separation lengths (should be naensgrp+nsclgrp-1)
-!      i_ensloccov4tim: flag of cross-temporal localization
-!                         =0: cross-temporal covariance is retained
-!                         =1: cross-temporal covariance is zero
-!      i_ensloccov4var: flag of cross-variable localization
-!                         =0: cross-variable covariance is retained
-!                         =1: cross-variable covariance is zero
-!      i_ensloccov4scl: flag of cross-scale localization
-!                         =0: cross-scale covariance is retained
-!                         =1: cross-scale covariance is zero
+!      r_ensloccov4tim: factor multiplying to cross-time covariance
+!                         For example,
+!                         =0.0: cross-time covariance is decreased to zero
+!                         =0.5: cross-time covariance is decreased to half
+!                         =1.0: cross-time covariance is retained
+!      r_ensloccov4var: factor multiplying to cross-variable covariance
+!                         For example,
+!                         =0.0: cross-variable covariance is decreased to zero
+!                         =0.5: cross-variable covariance is decreased to half
+!                         =1.0: cross-variable covariance is retained
+!      r_ensloccov4scl: factor multiplying to cross-scale covariance
+!                         For example,
+!                         =0.0: cross-scale covariance is decreased to zero
+!                         =0.5: cross-scale covariance is decreased to half
+!                         =1.0: cross-scale covariance is retained
 !=====================================================================================================
 !
 !
@@ -326,7 +332,7 @@ module hybrid_ensemble_parameters
   public :: ensloccov4tim,ensloccov4var,ensloccov4scl
   public :: alphacvarsclgrpmat
   public :: l_timloc_opt
-  public :: i_ensloccov4tim,i_ensloccov4var,i_ensloccov4scl
+  public :: r_ensloccov4tim,r_ensloccov4var,r_ensloccov4scl
   public :: idaen3d,idaen2d
   public :: ens_fast_read
   public :: parallelization_over_ensmembers
@@ -395,9 +401,9 @@ module hybrid_ensemble_parameters
   integer(i_kind) :: ntotensgrp=1
   integer(i_kind) :: naensloc=1
   integer(i_kind) :: ngvarloc=1
-  integer(i_kind) :: i_ensloccov4tim=0
-  integer(i_kind) :: i_ensloccov4var=0
-  integer(i_kind) :: i_ensloccov4scl=0
+  real(r_kind) :: r_ensloccov4tim
+  real(r_kind) :: r_ensloccov4var
+  real(r_kind) :: r_ensloccov4scl
   integer(i_kind),allocatable,dimension(:) :: idaen3d,idaen2d
 
   real(r_kind),allocatable,dimension(:,:) :: spc_multwgt
@@ -501,6 +507,9 @@ subroutine init_hybrid_ensemble_parameters
   n_ens_fv3sar=0
   weight_ens_gfs=one
   weight_ens_fv3sar=one
+  r_ensloccov4tim=one
+  r_ensloccov4var=one
+  r_ensloccov4scl=one
   vdl_scale = 0
   vloc_varlist = 'aaa'
   global_spectral_filter_sd=.false.