From b01447c1b6ce66542029369a81b13d36bc799192 Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Mon, 22 Aug 2022 11:26:32 -0600 Subject: [PATCH 01/22] Per issue #2206, for the ECNT line type, added CRPS_EMP_FAIR. SL --- internal/test_unit/hdr/met_11_0.hdr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/test_unit/hdr/met_11_0.hdr b/internal/test_unit/hdr/met_11_0.hdr index 04059e10b1..99e888ee32 100644 --- a/internal/test_unit/hdr/met_11_0.hdr +++ b/internal/test_unit/hdr/met_11_0.hdr @@ -17,7 +17,7 @@ PJC : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_L PRC : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_THRESH _VAR_ PSTD : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_THRESH BASER BASER_NCL BASER_NCU RELIABILITY RESOLUTION UNCERTAINTY ROC_AUC BRIER BRIER_NCL BRIER_NCU BRIERCL BRIERCL_NCL BRIERCL_NCU BSS BSS_SMPL _VAR_ ECLV : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL BASE N_PTS _VAR_ -ECNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_ENS CRPS CRPSS IGN ME RMSE SPREAD ME_OERR RMSE_OERR SPREAD_OERR SPREAD_PLUS_OERR CRPSCL CRPS_EMP CRPSCL_EMP CRPSS_EMP +ECNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_ENS CRPS CRPSS IGN ME RMSE SPREAD ME_OERR RMSE_OERR SPREAD_OERR SPREAD_PLUS_OERR CRPSCL CRPS_EMP CRPSCL_EMP CRPSS_EMP CRPS_EMP_FAIR RPS : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_PROB RPS_REL RPS_RES RPS_UNC RPS RPSS RPSS_SMPL RPS_COMP RHIST : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL CRPS IGN N_RANK CRPSS SPREAD _VAR_ PHIST : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL BIN_SIZE N_BIN _VAR_ From 7b473f4102b576d04416585c590c7bb27be6ed6a Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Mon, 22 Aug 2022 11:29:10 -0600 Subject: [PATCH 02/22] Per issue #2206, for ECNT line type, added CRPS_EMP_FAIR. SL --- data/table_files/met_header_columns_V11.0.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/table_files/met_header_columns_V11.0.txt b/data/table_files/met_header_columns_V11.0.txt index 9ef8beac4c..d57e089cb0 100644 --- a/data/table_files/met_header_columns_V11.0.txt +++ b/data/table_files/met_header_columns_V11.0.txt @@ -17,7 +17,7 @@ V11.0 : STAT : PJC : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID V11.0 : STAT : PRC : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL (N_THRESH) THRESH_[0-9]* PODY_[0-9]* POFD_[0-9]* V11.0 : STAT : PSTD : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL (N_THRESH) BASER BASER_NCL BASER_NCU RELIABILITY RESOLUTION UNCERTAINTY ROC_AUC BRIER BRIER_NCL BRIER_NCU BRIERCL BRIERCL_NCL BRIERCL_NCU BSS BSS_SMPL THRESH_[0-9]* V11.0 : STAT : ECLV : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL BASER VALUE_BASER (N_PTS) CL_[0-9]* VALUE_[0-9]* -V11.0 : STAT : ECNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_ENS CRPS CRPSS IGN ME RMSE SPREAD ME_OERR RMSE_OERR SPREAD_OERR SPREAD_PLUS_OERR CRPSCL CRPS_EMP CRPSCL_EMP CRPSS_EMP +V11.0 : STAT : ECNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_ENS CRPS CRPSS IGN ME RMSE SPREAD ME_OERR RMSE_OERR SPREAD_OERR SPREAD_PLUS_OERR CRPSCL CRPS_EMP CRPSCL_EMP CRPSS_EMP CRPS_EMP_FAIR V11.0 : STAT : RPS : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_PROB RPS_REL RPS_RES RPS_UNC RPS RPSS RPSS_SMPL RPS_COMP V11.0 : STAT : RHIST : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL (N_RANK) RANK_[0-9]* V11.0 : STAT : PHIST : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL BIN_SIZE (N_BIN) BIN_[0-9]* From 3a18c1d160ece3176b5c2d5cb022fb84b1a96c4e Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Mon, 22 Aug 2022 11:31:36 -0600 Subject: [PATCH 03/22] Per issue #2206, for ecnt_columns, added CRPS_EMP_FAIR. SL --- src/basic/vx_util/stat_column_defs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/basic/vx_util/stat_column_defs.h b/src/basic/vx_util/stat_column_defs.h index c76cf29b01..85273a1397 100644 --- a/src/basic/vx_util/stat_column_defs.h +++ b/src/basic/vx_util/stat_column_defs.h @@ -266,7 +266,7 @@ static const char * ecnt_columns [] = { "RMSE", "SPREAD", "ME_OERR", "RMSE_OERR", "SPREAD_OERR", "SPREAD_PLUS_OERR", "CRPSCL", "CRPS_EMP", "CRPSCL_EMP", - "CRPSS_EMP" + "CRPSS_EMP", "CRPS_EMP_FAIR" }; static const char * rps_columns [] = { From 017448caaa5deef0acc342934f956eaaa60c959a Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Mon, 22 Aug 2022 11:36:29 -0600 Subject: [PATCH 04/22] Per issue #2206, in write_ecnt_cols(), added crps_emp_fair. SL --- src/libcode/vx_stat_out/stat_columns.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libcode/vx_stat_out/stat_columns.cc b/src/libcode/vx_stat_out/stat_columns.cc index 1cdbfb5e53..f90a3f3ab7 100644 --- a/src/libcode/vx_stat_out/stat_columns.cc +++ b/src/libcode/vx_stat_out/stat_columns.cc @@ -4044,7 +4044,7 @@ void write_ecnt_cols(const ECNTInfo &ecnt_info, // RMSE, SPREAD, ME_OERR, // RMSE_OERR, SPREAD_OERR, SPREAD_PLUS_OERR, // CRPSCL, CRPS_EMP, CRPSCL_EMP, - // CRPSS_EMP + // CRPSS_EMP CRPS_EMP_FAIR // at.set_entry(r, c+0, // Total Number of Pairs ecnt_info.n_pair); @@ -4094,6 +4094,9 @@ void write_ecnt_cols(const ECNTInfo &ecnt_info, at.set_entry(r, c+15, // Empirical CRPSS ecnt_info.crpss_emp); + at.set_entry(r, c+16, // Empirical ensemble CRPS FAIR + ecnt_info.crps_emp_fair); + return; } From 35526446f5afba5ec1c90ab522faa7be9ad16ba0 Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Mon, 22 Aug 2022 11:42:36 -0600 Subject: [PATCH 05/22] Per issue #2206, in ECNTInfo, added crps_emp_fair. SL --- src/libcode/vx_statistics/ens_stats.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcode/vx_statistics/ens_stats.h b/src/libcode/vx_statistics/ens_stats.h index 7220487f57..a7cb7f972c 100644 --- a/src/libcode/vx_statistics/ens_stats.h +++ b/src/libcode/vx_statistics/ens_stats.h @@ -76,7 +76,7 @@ class ECNTInfo { // Number of ensemble members and pairs int n_ens, n_pair; - double crps_emp, crpscl_emp, crpss_emp; + double crps_emp, crpscl_emp, crpss_emp, crps_emp_fair; double crps_gaus, crpscl_gaus, crpss_gaus; double ign, me, rmse, spread; double me_oerr, rmse_oerr, spread_oerr; From faa6a1326e8b7720b4b7a525b2de8ece93073620 Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Wed, 24 Aug 2022 12:07:40 -0600 Subject: [PATCH 06/22] Per issue #2206, started stubbing code pieced to calculate crps_emp_fair. SL --- src/libcode/vx_statistics/pair_data_ensemble.cc | 16 +++++++++++++++- src/libcode/vx_statistics/pair_data_ensemble.h | 5 +++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/libcode/vx_statistics/pair_data_ensemble.cc b/src/libcode/vx_statistics/pair_data_ensemble.cc index 3c16d13033..a9df65d59e 100644 --- a/src/libcode/vx_statistics/pair_data_ensemble.cc +++ b/src/libcode/vx_statistics/pair_data_ensemble.cc @@ -99,6 +99,7 @@ void PairDataEnsemble::clear() { r_na.clear(); crps_emp_na.clear(); + crps_emp_fair_na.clear(); crpscl_emp_na.clear(); crps_gaus_na.clear(); crpscl_gaus_na.clear(); @@ -161,6 +162,7 @@ void PairDataEnsemble::extend(int n) { v_na.extend (n); r_na.extend (n); crps_emp_na.extend (n); + crps_emp_fair_na.extend (n); crpscl_emp_na.extend (n); crps_gaus_na.extend (n); crpscl_gaus_na.extend (n); @@ -219,6 +221,7 @@ void PairDataEnsemble::assign(const PairDataEnsemble &pd) { v_na = pd.v_na; r_na = pd.r_na; crps_emp_na = pd.crps_emp_na; + crps_emp_fair_na = pd.crps_emp_fair_na; crpscl_emp_na = pd.crpscl_emp_na; crps_gaus_na = pd.crps_gaus_na; crpscl_gaus_na = pd.crpscl_gaus_na; @@ -413,6 +416,7 @@ void PairDataEnsemble::compute_pair_vals(const gsl_rng *rng_ptr) { var_plus_oerr_na.add(bad_data_double); r_na.add(bad_data_int); crps_emp_na.add(bad_data_double); + crps_emp_fair_na.add(bad_data_double); crpscl_emp_na.add(bad_data_double); crps_gaus_na.add(bad_data_double); crpscl_gaus_na.add(bad_data_double); @@ -472,6 +476,15 @@ void PairDataEnsemble::compute_pair_vals(const gsl_rng *rng_ptr) { // Store empirical CRPS stats crps_emp_na.add(compute_crps_emp(o_na[i], cur_ens)); + + // Stub in for now? + // Like this? + //crps_emp_fair_na.add(compute_crps_emp(o_na[i], cur_ens) - compute_mean_abs_diff(cur_ens)); + + // Or like this? + // Note mean_abs_diff function should be added to NumArray class + crps_emp_fair_na.add( crps_emp_na[i] - ens_na[i].mean_abs_diff() ); + crpscl_emp_na.add(compute_crps_emp(o_na[i], cur_clm)); // Ensemble mean and standard deviation @@ -811,7 +824,7 @@ PairDataEnsemble PairDataEnsemble::subset_pairs_obs_thresh(const SingleThresh &o // // Include in subset: // wgt_na, o_na, cmn_na, csd_na, v_na, r_na, - // crps_emp_na, crpscl_emp_na, crps_gaus_na, crpscl_gaus_na, + // crps_emp_na, crps_emp_fair_na, crpscl_emp_na, crps_gaus_na, crpscl_gaus_na, // ign_na, pit_na, var_na, var_oerr_na, var_plus_oerr_na, // mn_na, mn_oerr_na, e_na // @@ -827,6 +840,7 @@ PairDataEnsemble PairDataEnsemble::subset_pairs_obs_thresh(const SingleThresh &o pd.v_na.add(v_na[i]); pd.r_na.add(r_na[i]); pd.crps_emp_na.add(crps_emp_na[i]); + pd.crps_emp_fair_na.add(crps_emp_fair_na[i]); pd.crpscl_emp_na.add(crpscl_emp_na[i]); pd.crps_gaus_na.add(crps_gaus_na[i]); pd.crpscl_gaus_na.add(crpscl_gaus_na[i]); diff --git a/src/libcode/vx_statistics/pair_data_ensemble.h b/src/libcode/vx_statistics/pair_data_ensemble.h index 86ad381395..ce4210c0a6 100644 --- a/src/libcode/vx_statistics/pair_data_ensemble.h +++ b/src/libcode/vx_statistics/pair_data_ensemble.h @@ -79,8 +79,9 @@ class PairDataEnsemble : public PairBase { NumArray v_na; // Number of valid ensemble values [n_obs] NumArray r_na; // Observation ranks [n_obs] - NumArray crps_emp_na; // Empirical Continuous Ranked Probability Score [n_obs] - NumArray crpscl_emp_na; // Empirical climatological CRPS [n_obs] + NumArray crps_emp_na; // Empirical Continuous Ranked Probability Score [n_obs] + NumArray crps_emp_fair_na; // Empirical Continuous Ranked Probability Score [n_obs], adjusted with the mean absolute difference of the ensmble members + NumArray crpscl_emp_na; // Empirical climatological CRPS [n_obs] NumArray crps_gaus_na; // Gaussian CRPS [n_obs] NumArray crpscl_gaus_na; // Gaussian climatological CRPS [n_obs] From 53c72f57ea531e95baa420c9ec2de08ea6a8829f Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Wed, 24 Aug 2022 14:36:39 -0600 Subject: [PATCH 07/22] Per issue #2206, added new function mean_abs_diff(). SL --- src/basic/vx_util/num_array.cc | 29 +++++++++++++++++++++++++++++ src/basic/vx_util/num_array.h | 3 ++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/basic/vx_util/num_array.cc b/src/basic/vx_util/num_array.cc index 002e1b5b61..d4f3a86a35 100644 --- a/src/basic/vx_util/num_array.cc +++ b/src/basic/vx_util/num_array.cc @@ -1273,4 +1273,33 @@ ConcatString write_css_hhmmss(const NumArray &na) } +//////////////////////////////////////////////////////////////////////// + + +double NumArray::mean_abs_diff() const + +{ + + int i, j, count; + double sum, mad; + + int n = n_elements(); + + for(i=0, count=0, sum=0.0; i Date: Thu, 25 Aug 2022 10:55:43 -0600 Subject: [PATCH 08/22] Per issue #2206, in compute_pair_vals() updated the code to calculate crps_emp and crps_emp_fair. SL --- src/libcode/vx_statistics/pair_data_ensemble.cc | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/libcode/vx_statistics/pair_data_ensemble.cc b/src/libcode/vx_statistics/pair_data_ensemble.cc index a9df65d59e..b376641dda 100644 --- a/src/libcode/vx_statistics/pair_data_ensemble.cc +++ b/src/libcode/vx_statistics/pair_data_ensemble.cc @@ -475,16 +475,11 @@ void PairDataEnsemble::compute_pair_vals(const gsl_rng *rng_ptr) { derive_climo_vals(cdf_info_ptr, cmn_na[i], csd_na[i], cur_clm); // Store empirical CRPS stats - crps_emp_na.add(compute_crps_emp(o_na[i], cur_ens)); - - // Stub in for now? - // Like this? - //crps_emp_fair_na.add(compute_crps_emp(o_na[i], cur_ens) - compute_mean_abs_diff(cur_ens)); - - // Or like this? - // Note mean_abs_diff function should be added to NumArray class - crps_emp_fair_na.add( crps_emp_na[i] - ens_na[i].mean_abs_diff() ); - + // + // For crps_emp use temporary, local variable so we can use it for the crps_emp_fair calculation + double crps_emp = compute_crps_emp(o_na[i], cur_ens); + crps_emp_na.add(crps_emp); + crps_emp_fair_na.add(crps_emp - cur_ens.mean_abs_diff()); crpscl_emp_na.add(compute_crps_emp(o_na[i], cur_clm)); // Ensemble mean and standard deviation From 7ffe9b47516697ac08f3378be29cd7aba6dde288 Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Mon, 29 Aug 2022 10:46:26 -0600 Subject: [PATCH 09/22] Per issue #2206, added crps_emp_fair (CRPS_EMP_FAIR) to relevant code pieces. SL --- src/tools/core/stat_analysis/aggr_stat_line.cc | 10 +++++++--- src/tools/core/stat_analysis/parse_stat_line.cc | 1 + src/tools/core/stat_analysis/parse_stat_line.h | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/tools/core/stat_analysis/aggr_stat_line.cc b/src/tools/core/stat_analysis/aggr_stat_line.cc index 35031d8720..e94fa81bf8 100644 --- a/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -2568,7 +2568,7 @@ void aggr_ecnt_lines(LineDataFile &f, STATAnalysisJob &job, AggrENSInfo aggr; ECNTData cur; ConcatString key; - double crps_emp, crpscl_emp, crps_gaus, crpscl_gaus, v; + double crps_emp, crps_emp_fair, crpscl_emp, crps_gaus, crpscl_gaus, v; map::iterator it; // @@ -2627,6 +2627,7 @@ void aggr_ecnt_lines(LineDataFile &f, STATAnalysisJob &job, // Store the current statistics and weight (TOTAL column) // m[key].ens_pd.crps_emp_na.add(cur.crps_emp); + m[key].ens_pd.crps_emp_fair_na.add(cur.crps_emp_fair); m[key].ens_pd.crpscl_emp_na.add(cur.crpscl_emp); m[key].ens_pd.crps_gaus_na.add(cur.crps_gaus); m[key].ens_pd.crpscl_gaus_na.add(cur.crpscl_gaus); @@ -2675,6 +2676,7 @@ void aggr_ecnt_lines(LineDataFile &f, STATAnalysisJob &job, it->second.ens_pd.rmse_oerr = (is_bad_data(v) ? bad_data_double : sqrt(v)); crps_emp = it->second.ens_pd.crps_emp_na.wmean(it->second.ens_pd.wgt_na); + crps_emp_fair = it->second.ens_pd.crps_emp_fair_na.wmean(it->second.ens_pd.wgt_na); crpscl_emp = it->second.ens_pd.crpscl_emp_na.wmean(it->second.ens_pd.wgt_na); crps_gaus = it->second.ens_pd.crps_gaus_na.wmean(it->second.ens_pd.wgt_na); crpscl_gaus = it->second.ens_pd.crpscl_gaus_na.wmean(it->second.ens_pd.wgt_na); @@ -3181,8 +3183,10 @@ void aggr_orank_lines(LineDataFile &f, STATAnalysisJob &job, derive_climo_vals(&m[key].cdf_info, cur.climo_mean, cur.climo_stdev, climo_vals); - // Store empirical CRPS stats - m[key].ens_pd.crps_emp_na.add(compute_crps_emp(cur.obs, cur.ens_na)); + // Store empirical CRPS stats and CRPS-Fair + double crps_emp = compute_crps_emp(cur.obs, cur.ens_na); + m[key].ens_pd.crps_emp_na.add(crps_emp); + m[key].ens_pd.crps_emp_fair_na.add(crps_emp - cur.ens_na.mean_abs_diff()); m[key].ens_pd.crpscl_emp_na.add(compute_crps_emp(cur.obs, climo_vals)); // Store Gaussian CRPS stats diff --git a/src/tools/core/stat_analysis/parse_stat_line.cc b/src/tools/core/stat_analysis/parse_stat_line.cc index b96d012e75..bf0c7fbc16 100644 --- a/src/tools/core/stat_analysis/parse_stat_line.cc +++ b/src/tools/core/stat_analysis/parse_stat_line.cc @@ -358,6 +358,7 @@ void parse_ecnt_line(STATLine &l, ECNTData &e_data) { e_data.n_ens = atof(l.get_item("N_ENS")); e_data.crps_emp = atof(l.get_item("CRPS_EMP")); + e_data.crps_emp_fair = atof(l.get_item("CRPS_EMP_FAIR")); e_data.crpscl_emp = atof(l.get_item("CRPSCL_EMP")); e_data.crpss_emp = atof(l.get_item("CRPSS_EMP")); diff --git a/src/tools/core/stat_analysis/parse_stat_line.h b/src/tools/core/stat_analysis/parse_stat_line.h index eed5c91eb8..ed9a6be48a 100644 --- a/src/tools/core/stat_analysis/parse_stat_line.h +++ b/src/tools/core/stat_analysis/parse_stat_line.h @@ -62,7 +62,7 @@ struct MPRData { // Ensemble continuous statistics (ECNT) data structure struct ECNTData { int total, n_ens; - double crps_emp, crpscl_emp, crpss_emp; + double crps_emp, crps_emp_fair, crpscl_emp, crpss_emp; double crps_gaus, crpscl_gaus, crpss_gaus; double ign, me, rmse, spread; double me_oerr, rmse_oerr, spread_oerr; From 1eeed47b6be4a22bf0e0762734f713dcc4e34e12 Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Mon, 29 Aug 2022 15:25:03 -0600 Subject: [PATCH 10/22] Per issue #2206, added wording for CRPS_EMP_FAIR, also added it to last column of ECNT line type. SL --- docs/Users_Guide/ensemble-stat.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/Users_Guide/ensemble-stat.rst b/docs/Users_Guide/ensemble-stat.rst index e989913062..6eb9f008d3 100644 --- a/docs/Users_Guide/ensemble-stat.rst +++ b/docs/Users_Guide/ensemble-stat.rst @@ -36,7 +36,7 @@ Often, the goal of ensemble forecasting is to reproduce the distribution of obse The relative position (RELP) is a count of the number of times each ensemble member is closest to the observation. For stochastic or randomly derived ensembles, this statistic is meaningless. For specified ensemble members, however, it can assist users in determining if any ensemble member is performing consistently better or worse than the others. -The ranked probability score (RPS) is included in the Ranked Probability Score (RPS) line type. It is the mean of the Brier scores computed from ensemble probabilities derived for each probability category threshold (prob_cat_thresh) specified in the configuration file. The continuous ranked probability score (CRPS) is the average the distance between the forecast (ensemble) cumulative distribution function and the observation cumulative distribution function. It is an analog of the Brier score, but for continuous forecast and observation fields. The CRPS statistic is computed using two methods: assuming a normal distribution defined by the ensemble mean and spread (:ref:`Gneiting et al., 2004 `) and using the empirical ensemble distribution (:ref:`Hersbach, 2000 `). The CRPS statistic is included in the Ensemble Continuous Statistics (ECNT) line type, along with other statistics quantifying the ensemble spread and ensemble mean skill. +The ranked probability score (RPS) is included in the Ranked Probability Score (RPS) line type. It is the mean of the Brier scores computed from ensemble probabilities derived for each probability category threshold (prob_cat_thresh) specified in the configuration file. The continuous ranked probability score (CRPS) is the average the distance between the forecast (ensemble) cumulative distribution function and the observation cumulative distribution function. It is an analog of the Brier score, but for continuous forecast and observation fields. The CRPS statistic is computed using two methods: assuming a normal distribution defined by the ensemble mean and spread (:ref:`Gneiting et al., 2004 `) and using the empirical ensemble distribution (:ref:`Hersbach, 2000 `). The CRPS statistic using the empirical ensemble distribution can be adjusted (bias corrected) by subtracting 1/2(m) times the mean absolute difference of the ensemble members (where m is the ensemble size), this is saved as a separate stastics called CRPS_EMP_FAIR. The CRPS (and CRPS FAIR) statistic is included in the Ensemble Continuous Statistics (ECNT) line type, along with other statistics quantifying the ensemble spread and ensemble mean skill. The Ensemble-Stat tool can derive ensemble relative frequencies and verify them as probability forecasts all in the same run. Note however that these simple ensemble relative frequencies are not actually calibrated probability forecasts. If probabilistic line types are requested (output_flag), this logic is applied to each pair of fields listed in the forecast (fcst) and observation (obs) dictionaries of the configuration file. Each probability category threshold (prob_cat_thresh) listed for the forecast field is applied to the input ensemble members to derive a relative frequency forecast. The probability category threshold (prob_cat_thresh) parsed from the corresponding observation entry is applied to the (gridded or point) observations to determine whether or not the event actually occurred. The paired ensemble relative freqencies and observation events are used to populate an Nx2 probabilistic contingency table. The dimension of that table is determined by the probability PCT threshold (prob_pct_thresh) configuration file option parsed from the forecast dictionary. All probabilistic output types requested are derived from the this Nx2 table and written to the ascii output files. Note that the FCST_VAR name header column is automatically reset as "PROB({FCST_VAR}{THRESH})" where {FCST_VAR} is the current field being evaluated and {THRESH} is the threshold that was applied. @@ -718,6 +718,9 @@ The format of the STAT and ASCII output of the Ensemble-Stat tool are described * - 40 - CRPSS_EMP - The Continuous Ranked Probability Skill Score (empirical distribution) + * - 41 + - CRPS_EMP_FAIR + - The Continuous Ranked Probability Skill Score (empirical distribution) adjusted by subtracting 1/2(m) times the mean absolute difference of the ensemble members (m is the ensemble size) .. _table_ES_header_info_es_out_RPS: From f4039c513dbcfb23a535c71500c506e23af7b2af Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Mon, 29 Aug 2022 15:50:22 -0600 Subject: [PATCH 11/22] Per issue #2206, added info for CRPS_EMP_FAIR and also added the math equation for this. SL --- docs/Users_Guide/appendixC.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/Users_Guide/appendixC.rst b/docs/Users_Guide/appendixC.rst index 0daab13dd0..c1f09aea48 100644 --- a/docs/Users_Guide/appendixC.rst +++ b/docs/Users_Guide/appendixC.rst @@ -972,9 +972,9 @@ where :math:`BS_m` is the Brier score for the m-th category (:ref:`Tödter and A CRPS ---- -Called "CRPS", "CRPSCL", "CRPS_EMP", and "CRPSCL_EMP" in ECNT output :numref:`table_ES_header_info_es_out_ECNT` +Called "CRPS", "CRPSCL", "CRPS_EMP", "CRPS_EMP_FAIR" and "CRPSCL_EMP" in ECNT output :numref:`table_ES_header_info_es_out_ECNT` -The continuous ranked probability score (CRPS) is the integral, over all possible thresholds, of the Brier scores (:ref:`Gneiting et al., 2004 `). In MET, the CRPS is calculated two ways: using a normal distribution fit to the ensemble forecasts (CRPS and CRPSCL), and using the empirical ensemble distribution (CRPS_EMP and CRPSCL_EMP). In some cases, use of other distributions would be better. +The continuous ranked probability score (CRPS) is the integral, over all possible thresholds, of the Brier scores (:ref:`Gneiting et al., 2004 `). In MET, the CRPS is calculated two ways: using a normal distribution fit to the ensemble forecasts (CRPS and CRPSCL), and using the empirical ensemble distribution (CRPS_EMP and CRPSCL_EMP). The empirical ensemble CRPS is also adjusted by subtracting 1/2(m) times the mean absolute difference of the ensemble members (where m is the ensemble size), this is saved as CRPS_EMP_FAIR. In some cases, use of other distributions would be better. WARNING: The normal distribution is probably a good fit for temperature and pressure, and possibly a not horrible fit for winds. However, the normal approximation will not work on most precipitation forecasts and may fail for many other atmospheric variables. @@ -988,6 +988,10 @@ The overall CRPS is calculated as the average of the individual measures. In equ The score can be interpreted as a continuous version of the mean absolute error (MAE). Thus, the score is negatively oriented, so smaller is better. Further, similar to MAE, bias will inflate the CRPS. Thus, bias should also be calculated and considered when judging forecast quality using CRPS. +To calculate CRPS_EMP_FAIR (bias adjusted, empirical ensemble CRPS) + +.. math:: `\text{CRPS_EMP_FAR} = CRPS_EMP - \frac{1}{n*(n-1)} \sum|f_{i} - f_{j}|`. + CRPS Skill Score ---------------- From 6648bcdbfb272befa634b28882010b479294b27c Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Tue, 30 Aug 2022 08:25:52 -0600 Subject: [PATCH 12/22] Per issue #2206, update the math equation for CRPS_EMP_FAIR. SL --- docs/Users_Guide/appendixC.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Users_Guide/appendixC.rst b/docs/Users_Guide/appendixC.rst index c1f09aea48..12e8aa45a0 100644 --- a/docs/Users_Guide/appendixC.rst +++ b/docs/Users_Guide/appendixC.rst @@ -990,8 +990,8 @@ The score can be interpreted as a continuous version of the mean absolute error To calculate CRPS_EMP_FAIR (bias adjusted, empirical ensemble CRPS) -.. math:: `\text{CRPS_EMP_FAR} = CRPS_EMP - \frac{1}{n*(n-1)} \sum|f_{i} - f_{j}|`. - +.. math:: \text{CRPS_EMP_FAIR} = \text{CRPS_EMP} - \frac{1}{n*(n-1)} \sum|f_{i} - f_{j}| + CRPS Skill Score ---------------- From 2c13f4a5a2853df2d3af0be7b5833b447143e704 Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Wed, 7 Sep 2022 14:34:50 -0600 Subject: [PATCH 13/22] Per issue #2206, added new function weighted_mean_absolute_diff(). SL --- src/basic/vx_util/num_array.cc | 80 ++++++++++++++++++++++------------ src/basic/vx_util/num_array.h | 1 + 2 files changed, 52 insertions(+), 29 deletions(-) diff --git a/src/basic/vx_util/num_array.cc b/src/basic/vx_util/num_array.cc index d4f3a86a35..8cec3bef2d 100644 --- a/src/basic/vx_util/num_array.cc +++ b/src/basic/vx_util/num_array.cc @@ -1233,24 +1233,29 @@ double NumArray::stdev(int skip_index) const //////////////////////////////////////////////////////////////////////// -// -// Utility Functions -// - -//////////////////////////////////////////////////////////////////////// - -ConcatString write_css(const NumArray &na) +double NumArray::mean_abs_diff() const { - ConcatString css; + int i, j, count; + double sum, mad; - for ( int i=0; i Date: Wed, 7 Sep 2022 14:49:37 -0600 Subject: [PATCH 14/22] Per issue #2206, updated the crps_emp_fair calculation to subtract the weighted_mean_abs_diff. SL --- src/libcode/vx_statistics/pair_data_ensemble.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcode/vx_statistics/pair_data_ensemble.cc b/src/libcode/vx_statistics/pair_data_ensemble.cc index b376641dda..be94fab5ca 100644 --- a/src/libcode/vx_statistics/pair_data_ensemble.cc +++ b/src/libcode/vx_statistics/pair_data_ensemble.cc @@ -479,7 +479,7 @@ void PairDataEnsemble::compute_pair_vals(const gsl_rng *rng_ptr) { // For crps_emp use temporary, local variable so we can use it for the crps_emp_fair calculation double crps_emp = compute_crps_emp(o_na[i], cur_ens); crps_emp_na.add(crps_emp); - crps_emp_fair_na.add(crps_emp - cur_ens.mean_abs_diff()); + crps_emp_fair_na.add(crps_emp - cur_ens.weighted_mean_abs_diff()); crpscl_emp_na.add(compute_crps_emp(o_na[i], cur_clm)); // Ensemble mean and standard deviation From 44309c691017b3101abf732ed20e982a80fa044f Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Wed, 7 Sep 2022 15:00:14 -0600 Subject: [PATCH 15/22] Per issue #2206, in aggr_orank_lines(), updated calculation for crps_emp_fair to use weighted_mean_abs_diff. SL --- src/tools/core/stat_analysis/aggr_stat_line.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/core/stat_analysis/aggr_stat_line.cc b/src/tools/core/stat_analysis/aggr_stat_line.cc index e94fa81bf8..bb64bfb40c 100644 --- a/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -3186,7 +3186,7 @@ void aggr_orank_lines(LineDataFile &f, STATAnalysisJob &job, // Store empirical CRPS stats and CRPS-Fair double crps_emp = compute_crps_emp(cur.obs, cur.ens_na); m[key].ens_pd.crps_emp_na.add(crps_emp); - m[key].ens_pd.crps_emp_fair_na.add(crps_emp - cur.ens_na.mean_abs_diff()); + m[key].ens_pd.crps_emp_fair_na.add(crps_emp - cur.ens_na.weighted_mean_abs_diff()); m[key].ens_pd.crpscl_emp_na.add(compute_crps_emp(cur.obs, climo_vals)); // Store Gaussian CRPS stats From 344289773dd80ee7353437e615495565528ae830 Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Thu, 8 Sep 2022 09:26:12 -0600 Subject: [PATCH 16/22] Per issue #2206, renamed weighted_mean_abs_diff() to wmean_abs_diff(). SL --- src/basic/vx_util/num_array.cc | 12 +++++------- src/basic/vx_util/num_array.h | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/basic/vx_util/num_array.cc b/src/basic/vx_util/num_array.cc index 8cec3bef2d..05cd941ff6 100644 --- a/src/basic/vx_util/num_array.cc +++ b/src/basic/vx_util/num_array.cc @@ -600,8 +600,6 @@ void NumArray::reorder(const NumArray &i_na) { return; } -// SETH, please review the logic of the functions below. -// Do they still work OK after switching to STL::vector? //////////////////////////////////////////////////////////////////////// // @@ -1263,22 +1261,22 @@ double NumArray::mean_abs_diff() const //////////////////////////////////////////////////////////////////////// -double NumArray::weighted_mean_abs_diff() const +double NumArray::wmean_abs_diff() const { - double mad, weighted_mad; + double mad, wmad; int n = n_elements(); mad = mean_abs_diff(); if( is_bad_data(mad) ) - weighted_mad = bad_data_double; + wmad = bad_data_double; else - weighted_mad = (1/(2*n)) * mad; + wmad = (1/(2*n)) * mad; - return(weighted_mad); + return(wmad); } diff --git a/src/basic/vx_util/num_array.h b/src/basic/vx_util/num_array.h index 275dc2991b..9cb37243f6 100644 --- a/src/basic/vx_util/num_array.h +++ b/src/basic/vx_util/num_array.h @@ -110,7 +110,7 @@ class NumArray { double mean_sqrt() const; double mean_fisher() const; double mean_abs_diff() const; - double weighted_mean_abs_diff() const; + double wmean_abs_diff() const; double variance(int skip_index = bad_data_int) const; double stdev(int skip_index = bad_data_int) const; From 48648cfc9567386b41cdf837954791411665b1cb Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Thu, 8 Sep 2022 09:29:23 -0600 Subject: [PATCH 17/22] Per issue #2206, for the crps_emp_fair calculation, changed weighted_mean_abs_diff() to wmean_abs_diff(). SL --- src/libcode/vx_statistics/pair_data_ensemble.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcode/vx_statistics/pair_data_ensemble.cc b/src/libcode/vx_statistics/pair_data_ensemble.cc index be94fab5ca..4d8d2fccd4 100644 --- a/src/libcode/vx_statistics/pair_data_ensemble.cc +++ b/src/libcode/vx_statistics/pair_data_ensemble.cc @@ -479,7 +479,7 @@ void PairDataEnsemble::compute_pair_vals(const gsl_rng *rng_ptr) { // For crps_emp use temporary, local variable so we can use it for the crps_emp_fair calculation double crps_emp = compute_crps_emp(o_na[i], cur_ens); crps_emp_na.add(crps_emp); - crps_emp_fair_na.add(crps_emp - cur_ens.weighted_mean_abs_diff()); + crps_emp_fair_na.add(crps_emp - cur_ens.wmean_abs_diff()); crpscl_emp_na.add(compute_crps_emp(o_na[i], cur_clm)); // Ensemble mean and standard deviation From c93090db0f019535b7ad46f7ce6616cac609ea09 Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Thu, 8 Sep 2022 09:32:23 -0600 Subject: [PATCH 18/22] Per issue #2206, in aggr_orank_lines(), for the crps_emp_fair calculation, changed weighted_mean_abs_diff() to wmean_abs_diff(). SL --- src/tools/core/stat_analysis/aggr_stat_line.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/core/stat_analysis/aggr_stat_line.cc b/src/tools/core/stat_analysis/aggr_stat_line.cc index bb64bfb40c..e2061d30c2 100644 --- a/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -3186,7 +3186,7 @@ void aggr_orank_lines(LineDataFile &f, STATAnalysisJob &job, // Store empirical CRPS stats and CRPS-Fair double crps_emp = compute_crps_emp(cur.obs, cur.ens_na); m[key].ens_pd.crps_emp_na.add(crps_emp); - m[key].ens_pd.crps_emp_fair_na.add(crps_emp - cur.ens_na.weighted_mean_abs_diff()); + m[key].ens_pd.crps_emp_fair_na.add(crps_emp - cur.ens_na.wmean_abs_diff()); m[key].ens_pd.crpscl_emp_na.add(compute_crps_emp(cur.obs, climo_vals)); // Store Gaussian CRPS stats From a451368d8f8a084cf21d43c455b0e637e44ccd2b Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Thu, 8 Sep 2022 10:19:43 -0600 Subject: [PATCH 19/22] Per issue #2206, updated the math equation for the CRPS_EMP_FAIR calculation. SL --- docs/Users_Guide/appendixC.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Users_Guide/appendixC.rst b/docs/Users_Guide/appendixC.rst index 12e8aa45a0..809b3bcb8b 100644 --- a/docs/Users_Guide/appendixC.rst +++ b/docs/Users_Guide/appendixC.rst @@ -990,7 +990,7 @@ The score can be interpreted as a continuous version of the mean absolute error To calculate CRPS_EMP_FAIR (bias adjusted, empirical ensemble CRPS) -.. math:: \text{CRPS_EMP_FAIR} = \text{CRPS_EMP} - \frac{1}{n*(n-1)} \sum|f_{i} - f_{j}| +.. math:: \text{CRPS_EMP_FAIR} = \text{CRPS_EMP} - \frac{1}{2*n} * \frac{1}{n*(n-1)} \sum|f_{i} - f_{j}| CRPS Skill Score ---------------- From ea9313159f1c7f1ca5c839a35337ea17055c872b Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Fri, 9 Sep 2022 11:02:31 -0600 Subject: [PATCH 20/22] Per issue #2206, fixed a bug in the wmean_abs_diff() function. SL --- src/basic/vx_util/num_array.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/basic/vx_util/num_array.cc b/src/basic/vx_util/num_array.cc index 05cd941ff6..e5b8adba2e 100644 --- a/src/basic/vx_util/num_array.cc +++ b/src/basic/vx_util/num_array.cc @@ -1265,16 +1265,16 @@ double NumArray::wmean_abs_diff() const { - double mad, wmad; + double wmad; int n = n_elements(); + double wgt = 1.0/(2.0*n); + double mad = mean_abs_diff(); - mad = mean_abs_diff(); - - if( is_bad_data(mad) ) + if( is_bad_data(mad) ) wmad = bad_data_double; - else - wmad = (1/(2*n)) * mad; + else + wmad = wgt * mad; return(wmad); From 32c060a14d99655c0654ca1cc9069bf11186077b Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Mon, 12 Sep 2022 09:55:02 -0600 Subject: [PATCH 21/22] Per issue #2206, added code for crps_emp_fair. SL --- src/libcode/vx_statistics/ens_stats.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/libcode/vx_statistics/ens_stats.cc b/src/libcode/vx_statistics/ens_stats.cc index d7c0e2c088..e9811b5fd3 100644 --- a/src/libcode/vx_statistics/ens_stats.cc +++ b/src/libcode/vx_statistics/ens_stats.cc @@ -177,7 +177,7 @@ void ECNTInfo::clear() { othresh.clear(); n_ens = n_pair = 0; - crps_emp = crpscl_emp = crpss_emp = bad_data_double; + crps_emp = crpscl_emp = crpss_emp = crps_emp_fair = bad_data_double; crps_gaus = crpscl_gaus = crpss_gaus = bad_data_double; ign = bad_data_double; me = rmse = spread = bad_data_double; @@ -199,7 +199,8 @@ void ECNTInfo::assign(const ECNTInfo &c) { crps_emp = c.crps_emp; crpscl_emp = c.crpscl_emp; crpss_emp = c.crpss_emp; - + crps_emp_fair = c.crps_emp_fair; + crps_gaus = c.crps_gaus; crpscl_gaus = c.crpscl_gaus; crpss_gaus = c.crpss_gaus; @@ -230,6 +231,8 @@ void ECNTInfo::set(const PairDataEnsemble &pd) { // Compute empirical CRPS scores crps_emp = pd.crps_emp_na.wmean(pd.wgt_na); + // Stub in for now? + crps_emp_fair = pd.crps_emp_fair_na.wmean(pd.wgt_na); crpscl_emp = pd.crpscl_emp_na.wmean(pd.wgt_na); crpss_emp = (is_bad_data(crps_emp) || is_bad_data(crpscl_emp) || From f9e17db955bc0b9acc9bc2685853a68278900668 Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Mon, 12 Sep 2022 10:26:21 -0600 Subject: [PATCH 22/22] Per issue #2206, in set() function, added checks for crps_emp and crps_emp_fair, for values of 0.0. SL --- src/libcode/vx_statistics/ens_stats.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libcode/vx_statistics/ens_stats.cc b/src/libcode/vx_statistics/ens_stats.cc index e9811b5fd3..3167c814b9 100644 --- a/src/libcode/vx_statistics/ens_stats.cc +++ b/src/libcode/vx_statistics/ens_stats.cc @@ -231,8 +231,11 @@ void ECNTInfo::set(const PairDataEnsemble &pd) { // Compute empirical CRPS scores crps_emp = pd.crps_emp_na.wmean(pd.wgt_na); - // Stub in for now? + if(is_eq(crps_emp, 0.0)) crps_emp = 0.0; + crps_emp_fair = pd.crps_emp_fair_na.wmean(pd.wgt_na); + if(is_eq(crps_emp_fair, 0.0)) crps_emp_fair = 0.0; + crpscl_emp = pd.crpscl_emp_na.wmean(pd.wgt_na); crpss_emp = (is_bad_data(crps_emp) || is_bad_data(crpscl_emp) ||