diff --git a/data/table_files/met_header_columns_V11.0.txt b/data/table_files/met_header_columns_V11.0.txt index e7a896a335..2c9cf33896 100644 --- a/data/table_files/met_header_columns_V11.0.txt +++ b/data/table_files/met_header_columns_V11.0.txt @@ -19,7 +19,7 @@ V11.0 : STAT : PJC : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID V11.0 : STAT : PRC : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL (N_THRESH) THRESH_[0-9]* PODY_[0-9]* POFD_[0-9]* V11.0 : STAT : PSTD : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL (N_THRESH) BASER BASER_NCL BASER_NCU RELIABILITY RESOLUTION UNCERTAINTY ROC_AUC BRIER BRIER_NCL BRIER_NCU BRIERCL BRIERCL_NCL BRIERCL_NCU BSS BSS_SMPL THRESH_[0-9]* V11.0 : STAT : ECLV : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL BASER VALUE_BASER (N_PTS) CL_[0-9]* VALUE_[0-9]* -V11.0 : STAT : ECNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_ENS CRPS CRPSS IGN ME RMSE SPREAD ME_OERR RMSE_OERR SPREAD_OERR SPREAD_PLUS_OERR CRPSCL CRPS_EMP CRPSCL_EMP CRPSS_EMP CRPS_EMP_FAIR MAE MAE_OERR BIAS_RATIO N_GE_OBS ME_GE_OBS N_LT_OBS ME_LT_OBS +V11.0 : STAT : ECNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_ENS CRPS CRPSS IGN ME RMSE SPREAD ME_OERR RMSE_OERR SPREAD_OERR SPREAD_PLUS_OERR CRPSCL CRPS_EMP CRPSCL_EMP CRPSS_EMP CRPS_EMP_FAIR SPREAD_MD MAE MAE_OERR BIAS_RATIO N_GE_OBS ME_GE_OBS N_LT_OBS ME_LT_OBS V11.0 : STAT : RPS : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_PROB RPS_REL RPS_RES RPS_UNC RPS RPSS RPSS_SMPL RPS_COMP V11.0 : STAT : RHIST : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL (N_RANK) RANK_[0-9]* V11.0 : STAT : PHIST : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL BIN_SIZE (N_BIN) BIN_[0-9]* diff --git a/docs/Users_Guide/appendixC.rst b/docs/Users_Guide/appendixC.rst index f40e779ebe..0ac8b09cbf 100644 --- a/docs/Users_Guide/appendixC.rst +++ b/docs/Users_Guide/appendixC.rst @@ -977,7 +977,7 @@ CRPS Called "CRPS", "CRPSCL", "CRPS_EMP", "CRPS_EMP_FAIR" and "CRPSCL_EMP" in ECNT output :numref:`table_ES_header_info_es_out_ECNT` -The continuous ranked probability score (CRPS) is the integral, over all possible thresholds, of the Brier scores (:ref:`Gneiting et al., 2004 `). In MET, the CRPS is calculated two ways: using a normal distribution fit to the ensemble forecasts (CRPS and CRPSCL), and using the empirical ensemble distribution (CRPS_EMP and CRPSCL_EMP). The empirical ensemble CRPS is also adjusted by subtracting 1/2(m) times the mean absolute difference of the ensemble members (where m is the ensemble size), this is saved as CRPS_EMP_FAIR. In some cases, use of other distributions would be better. +The continuous ranked probability score (CRPS) is the integral, over all possible thresholds, of the Brier scores (:ref:`Gneiting et al., 2004 `). In MET, the CRPS is calculated two ways: using a normal distribution fit to the ensemble forecasts (CRPS and CRPSCL), and using the empirical ensemble distribution (CRPS_EMP and CRPSCL_EMP). The empirical ensemble CRPS can be adjusted (bias corrected) by subtracting 1/(2*m) times the mean absolute difference of the ensemble members, where m is the ensemble size. This is reported as a separate statistic called CRPS_EMP_FAIR. In some cases, use of other distributions would be better. WARNING: The normal distribution is probably a good fit for temperature and pressure, and possibly a not horrible fit for winds. However, the normal approximation will not work on most precipitation forecasts and may fail for many other atmospheric variables. @@ -987,13 +987,34 @@ Closed form expressions for the CRPS are difficult to define when using data rat In this equation, the y represents the event threshold. The estimated mean and standard deviation of the ensemble forecasts ( :math:`\mu \text{ and } \sigma`) are used as the parameters of the normal distribution. The values of the normal distribution are represented by the probability density function (PDF) denoted by :math:`\Phi` and the cumulative distribution function (CDF), denoted in the above equation by :math:`\phi`. -The overall CRPS is calculated as the average of the individual measures. In equation form: :math:`\text{CRPS} = \text{average(crps) } = \frac{1}{N} \sum_i^N \text{crps}_i`. +The overall CRPS is calculated as the average of the individual measures. In equation form: + +.. math:: \text{CRPS} = \text{average(crps) } = \frac{1}{N} \sum_{i=1}^N \text{crps}_i The score can be interpreted as a continuous version of the mean absolute error (MAE). Thus, the score is negatively oriented, so smaller is better. Further, similar to MAE, bias will inflate the CRPS. Thus, bias should also be calculated and considered when judging forecast quality using CRPS. -To calculate CRPS_EMP_FAIR (bias adjusted, empirical ensemble CRPS) +To calculate crps_emp_fair (bias adjusted, empirical ensemble CRPS) for each individual observation with m ensemble members: + +.. math:: \text{crps_emp_fair}_i = \text{crps_emp}_i - \frac{1}{2*m} * \frac{1}{m*(m-1)} \sum_{i \ne j}|f_{i} - f_{j}| + +The overall CRPS_EMP_FAIR is calculated as the average of the individual measures. In equation form: + +.. math:: \text{CRPS_EMP_FAIR} = \text{average(crps_emp_fair) } = \frac{1}{N} \sum_{i=1}^N \text{crps_emp_fair}_i + +Ensemble Mean Absolute Difference +--------------------------------- + +Called "SPREAD_MD" in ECNT output :numref:`table_ES_header_info_es_out_ECNT` + +The ensemble mean absolute difference is an alternative measure of ensemble spread. It is computed for each individual observation (denoted by a lowercase spread_md) with m ensemble members: + +.. math:: \text{spread_md}_i = \frac{1}{m*(m-1)} \sum_{i \ne j}|f_{i} - f_{j}| + +The overall SPREAD_MD is calculated as the average of the individual measures. In equation form: + +.. math:: \text{SPREAD_MD} = \text{average(spread_md) } = \frac{1}{N} \sum_{i=1}^N \text{spread_md}_i -.. math:: \text{CRPS_EMP_FAIR} = \text{CRPS_EMP} - \frac{1}{2*n} * \frac{1}{n*(n-1)} \sum|f_{i} - f_{j}| +A perfect forecast would have ensemble mean absolute difference = 0. CRPS Skill Score ---------------- diff --git a/docs/Users_Guide/ensemble-stat.rst b/docs/Users_Guide/ensemble-stat.rst index de7b167edc..f8e6bfdcd8 100644 --- a/docs/Users_Guide/ensemble-stat.rst +++ b/docs/Users_Guide/ensemble-stat.rst @@ -48,7 +48,7 @@ Often, the goal of ensemble forecasting is to reproduce the distribution of obse The relative position (RELP) is a count of the number of times each ensemble member is closest to the observation. For stochastic or randomly derived ensembles, this statistic is meaningless. For specified ensemble members, however, it can assist users in determining if any ensemble member is performing consistently better or worse than the others. -The ranked probability score (RPS) is included in the Ranked Probability Score (RPS) line type. It is the mean of the Brier scores computed from ensemble probabilities derived for each probability category threshold (prob_cat_thresh) specified in the configuration file. The continuous ranked probability score (CRPS) is the average the distance between the forecast (ensemble) cumulative distribution function and the observation cumulative distribution function. It is an analog of the Brier score, but for continuous forecast and observation fields. The CRPS statistic is computed using two methods: assuming a normal distribution defined by the ensemble mean and spread (:ref:`Gneiting et al., 2004 `) and using the empirical ensemble distribution (:ref:`Hersbach, 2000 `). The CRPS statistic using the empirical ensemble distribution can be adjusted (bias corrected) by subtracting 1/2(m) times the mean absolute difference of the ensemble members (where m is the ensemble size), this is saved as a separate stastics called CRPS_EMP_FAIR. The CRPS (and CRPS FAIR) statistic is included in the Ensemble Continuous Statistics (ECNT) line type, along with other statistics quantifying the ensemble spread and ensemble mean skill. +The ranked probability score (RPS) is included in the Ranked Probability Score (RPS) line type. It is the mean of the Brier scores computed from ensemble probabilities derived for each probability category threshold (prob_cat_thresh) specified in the configuration file. The continuous ranked probability score (CRPS) is the average the distance between the forecast (ensemble) cumulative distribution function and the observation cumulative distribution function. It is an analog of the Brier score, but for continuous forecast and observation fields. The CRPS statistic is computed using two methods: assuming a normal distribution defined by the ensemble mean and spread (:ref:`Gneiting et al., 2004 `) and using the empirical ensemble distribution (:ref:`Hersbach, 2000 `). The CRPS statistic using the empirical ensemble distribution can be adjusted (bias corrected) by subtracting 1/(2*m) times the mean absolute difference of the ensemble members, where m is the ensemble size. This is reported as a separate statistic called CRPS_EMP_FAIR. The empirical CRPS and its fair version are included in the Ensemble Continuous Statistics (ECNT) line type, along with other statistics quantifying the ensemble spread and ensemble mean skill. The Ensemble-Stat tool can derive ensemble relative frequencies and verify them as probability forecasts all in the same run. Note however that these simple ensemble relative frequencies are not actually calibrated probability forecasts. If probabilistic line types are requested (output_flag), this logic is applied to each pair of fields listed in the forecast (fcst) and observation (obs) dictionaries of the configuration file. Each probability category threshold (prob_cat_thresh) listed for the forecast field is applied to the input ensemble members to derive a relative frequency forecast. The probability category threshold (prob_cat_thresh) parsed from the corresponding observation entry is applied to the (gridded or point) observations to determine whether or not the event actually occurred. The paired ensemble relative freqencies and observation events are used to populate an Nx2 probabilistic contingency table. The dimension of that table is determined by the probability PCT threshold (prob_pct_thresh) configuration file option parsed from the forecast dictionary. All probabilistic output types requested are derived from the this Nx2 table and written to the ascii output files. Note that the FCST_VAR name header column is automatically reset as "PROB({FCST_VAR}{THRESH})" where {FCST_VAR} is the current field being evaluated and {THRESH} is the threshold that was applied. @@ -622,26 +622,29 @@ The format of the STAT and ASCII output of the Ensemble-Stat tool are described - The Continuous Ranked Probability Skill Score (empirical distribution) * - 41 - CRPS_EMP_FAIR - - The Continuous Ranked Probability Skill Score (empirical distribution) adjusted by subtracting 1/2(m) times the mean absolute difference of the ensemble members (m is the ensemble size) + - The Continuous Ranked Probability Score (empirical distribution) adjusted by the mean absolute difference of the ensemble members * - 42 + - SPREAD_MD + - The pairwise Mean Absolute Difference of the unperturbed ensemble members + * - 43 - MAE - The Mean Absolute Error of the ensemble mean (unperturbed or supplied) - * - 43 + * - 44 - MAE_OERR - The Mean Absolute Error of the PERTURBED ensemble mean (e.g. with Observation Error) - * - 44 + * - 45 - BIAS_RATIO - The Bias Ratio - * - 45 + * - 46 - N_GE_OBS - The number of ensemble values greater than or equal to their observations - * - 46 + * - 47 - ME_GE_OBS - The Mean Error of the ensemble values greater than or equal to their observations - * - 47 + * - 48 - N_LT_OBS - The number of ensemble values less than their observations - * - 48 + * - 49 - ME_LT_OBS - The Mean Error of the ensemble values less than or equal to their observations diff --git a/docs/Users_Guide/point-stat.rst b/docs/Users_Guide/point-stat.rst index 9b9b9806cb..a26544ed19 100644 --- a/docs/Users_Guide/point-stat.rst +++ b/docs/Users_Guide/point-stat.rst @@ -866,13 +866,13 @@ The first set of header columns are common to all of the output files generated * - 113-115 - RMSFA, :raw-html:`
` RMSFA_BCL, :raw-html:`
` RMSFA_BCU - Root mean squared forecast anomaly (f-c) including bootstrap upper and lower confidence limits - * - 117-119 + * - 116-118 - RMSOA, :raw-html:`
` RMSOA_BCL, :raw-html:`
` RMSOA_BCU - Root mean squared observation anomaly (o-c) including bootstrap upper and lower confidence limits - * - 120-122 + * - 119-121 - ANOM_CORR_UNCNTR, :raw-html:`
` ANOM_CORR_UNCNTR_BCL, :raw-html:`
` ANOM_CORR_UNCNTR_BCU - The uncentered Anomaly Correlation excluding mean error including bootstrap upper and lower confidence limits - * - 123-125 + * - 122-124 - SI, :raw-html:`
` SI_BCL, :raw-html:`
` SI_BCU - Scatter Index including bootstrap upper and lower confidence limits diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index b3d654196c..91d48848f3 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -11,7 +11,7 @@ Important issues are listed **in bold** for emphasis. .. warning:: **Ensemble post-processing was added to Gen-Ens-Prod in version 10.1.0 and will be removed from Ensemble-Stat in version 11.0.0!** -MET Version 11.0.0-beta4 release notes (20221031) +MET Version 11.0.0-beta4 release notes (20221103) ------------------------------------------------- * Repository, build, and testing: @@ -44,6 +44,7 @@ MET Version 11.0.0-beta4 release notes (20221031) * **Enhance ASCII2NC to read NDBC buoy data** (`#2276 `_). * **Enhance IODA2NC to support IODA v2.0 format** (`#2068 `_). + * **Add the Mean Absolute Difference (SPREAD_MD) to the ECNT line type** (`#2332 `_). * **Add MAE to the ECNT line type from Ensemble-Stat and for HiRA** (`#2325 `_). * **Add new bias ratio statistic to the ECNT line type from Ensemble-Stat and for HiRA** (`#2058 `_). * **Enhance TC-Pairs consensus logic to compute the spread of the location, wind speed, and pressure** (`#2036 `_). diff --git a/docs/conf.py b/docs/conf.py index 3635edc6d6..039520f8fd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,7 @@ verinfo = version release = f'{version}' release_year = '2022' -release_date = f'{release_year}-10-31' +release_date = f'{release_year}-11-03' copyright = f'{release_year}, {author}' # -- General configuration --------------------------------------------------- diff --git a/internal/test_unit/hdr/met_11_0.hdr b/internal/test_unit/hdr/met_11_0.hdr index 2d08a8fbd5..c70e192768 100644 --- a/internal/test_unit/hdr/met_11_0.hdr +++ b/internal/test_unit/hdr/met_11_0.hdr @@ -17,7 +17,7 @@ PJC : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_L PRC : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_THRESH _VAR_ PSTD : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_THRESH BASER BASER_NCL BASER_NCU RELIABILITY RESOLUTION UNCERTAINTY ROC_AUC BRIER BRIER_NCL BRIER_NCU BRIERCL BRIERCL_NCL BRIERCL_NCU BSS BSS_SMPL _VAR_ ECLV : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL BASE N_PTS _VAR_ -ECNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_ENS CRPS CRPSS IGN ME RMSE SPREAD ME_OERR RMSE_OERR SPREAD_OERR SPREAD_PLUS_OERR CRPSCL CRPS_EMP CRPSCL_EMP CRPSS_EMP CRPS_EMP_FAIR MAE MAE_OERR BIAS_RATIO N_GE_OBS ME_GE_OBS N_LT_OBS ME_LT_OBS +ECNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_ENS CRPS CRPSS IGN ME RMSE SPREAD ME_OERR RMSE_OERR SPREAD_OERR SPREAD_PLUS_OERR CRPSCL CRPS_EMP CRPSCL_EMP CRPSS_EMP CRPS_EMP_FAIR SPREAD_MD MAE MAE_OERR BIAS_RATIO N_GE_OBS ME_GE_OBS N_LT_OBS ME_LT_OBS RPS : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_PROB RPS_REL RPS_RES RPS_UNC RPS RPSS RPSS_SMPL RPS_COMP RHIST : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL CRPS IGN N_RANK CRPSS SPREAD _VAR_ PHIST : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL BIN_SIZE N_BIN _VAR_ diff --git a/src/basic/vx_util/stat_column_defs.h b/src/basic/vx_util/stat_column_defs.h index a96fa2d26f..009ec547bb 100644 --- a/src/basic/vx_util/stat_column_defs.h +++ b/src/basic/vx_util/stat_column_defs.h @@ -267,9 +267,10 @@ static const char * ecnt_columns [] = { "RMSE", "SPREAD", "ME_OERR", "RMSE_OERR", "SPREAD_OERR", "SPREAD_PLUS_OERR", "CRPSCL", "CRPS_EMP", "CRPSCL_EMP", - "CRPSS_EMP", "CRPS_EMP_FAIR", "MAE", - "MAE_OERR", "BIAS_RATIO", "N_GE_OBS", - "ME_GE_OBS", "N_LT_OBS", "ME_LT_OBS" + "CRPSS_EMP", "CRPS_EMP_FAIR", "SPREAD_MD", + "MAE", "MAE_OERR", "BIAS_RATIO", + "N_GE_OBS", "ME_GE_OBS", "N_LT_OBS", + "ME_LT_OBS" }; static const char * rps_columns [] = { diff --git a/src/libcode/vx_stat_out/stat_columns.cc b/src/libcode/vx_stat_out/stat_columns.cc index 51ad6faee6..7844205758 100644 --- a/src/libcode/vx_stat_out/stat_columns.cc +++ b/src/libcode/vx_stat_out/stat_columns.cc @@ -4225,9 +4225,10 @@ void write_ecnt_cols(const ECNTInfo &ecnt_info, // RMSE, SPREAD, ME_OERR, // RMSE_OERR, SPREAD_OERR, SPREAD_PLUS_OERR, // CRPSCL, CRPS_EMP, CRPSCL_EMP, - // CRPSS_EMP CRPS_EMP_FAIR, MAE, - // MAE_OERR, BIAS RATIO, N_GE_OBS, - // ME_GE_OBS, N_LT_OBS, ME_LT_OBS + // CRPSS_EMP CRPS_EMP_FAIR, SPREAD_MD, + // MAE, MAE_OERR, BIAS_RATIO, + // N_GE_OBS, ME_GE_OBS, N_LT_OBS, + // ME_LT_OBS // at.set_entry(r, c+0, // Total Number of Pairs ecnt_info.n_pair); @@ -4280,25 +4281,28 @@ void write_ecnt_cols(const ECNTInfo &ecnt_info, at.set_entry(r, c+16, // Empirical ensemble CRPS FAIR ecnt_info.crps_emp_fair); - at.set_entry(r, c+17, // MAE for unperturbed ensemble mean + at.set_entry(r, c+17, // Mean absolute difference measure of spread + ecnt_info.spread_md); + + at.set_entry(r, c+18, // MAE for unperturbed ensemble mean ecnt_info.mae); - at.set_entry(r, c+18, // MAE for mean of perturbed members + at.set_entry(r, c+19, // MAE for mean of perturbed members ecnt_info.mae_oerr); - at.set_entry(r, c+19, // Bias Ratio + at.set_entry(r, c+20, // Bias Ratio ecnt_info.bias_ratio); - at.set_entry(r, c+20, // Number of ensemble values >= observations + at.set_entry(r, c+21, // Number of ensemble values >= observations ecnt_info.n_ge_obs); - at.set_entry(r, c+21, // ME of ensemble values >= observations + at.set_entry(r, c+22, // ME of ensemble values >= observations ecnt_info.me_ge_obs); - at.set_entry(r, c+22, // Number of ensemble values < observations + at.set_entry(r, c+23, // Number of ensemble values < observations ecnt_info.n_lt_obs); - at.set_entry(r, c+23, // ME of ensemble values < observations + at.set_entry(r, c+24, // ME of ensemble values < observations ecnt_info.me_lt_obs); return; diff --git a/src/libcode/vx_statistics/ens_stats.cc b/src/libcode/vx_statistics/ens_stats.cc index b64341a3dc..f0d28ac9c9 100644 --- a/src/libcode/vx_statistics/ens_stats.cc +++ b/src/libcode/vx_statistics/ens_stats.cc @@ -176,13 +176,14 @@ void ECNTInfo::init_from_scratch() { void ECNTInfo::clear() { othresh.clear(); - n_ens = n_pair = 0; - crps_emp = crpscl_emp = crpss_emp = crps_emp_fair = bad_data_double; - crps_gaus = crpscl_gaus = crpss_gaus = bad_data_double; - ign = bad_data_double; - me = mae = rmse = spread = bad_data_double; - me_oerr = mae_oerr = rmse_oerr = spread_oerr = bad_data_double; - spread_plus_oerr = bad_data_double; + n_ens = n_pair = 0; + crps_emp = crpscl_emp = crpss_emp = bad_data_double; + crps_emp_fair = spread_md = bad_data_double; + crps_gaus = crpscl_gaus = crpss_gaus = bad_data_double; + ign = bad_data_double; + me = mae = rmse = spread = bad_data_double; + me_oerr = mae_oerr = rmse_oerr = spread_oerr = bad_data_double; + spread_plus_oerr = bad_data_double; n_ge_obs = n_lt_obs = 0; me_ge_obs = me_lt_obs = bias_ratio = bad_data_double; @@ -203,6 +204,7 @@ void ECNTInfo::assign(const ECNTInfo &c) { crpscl_emp = c.crpscl_emp; crpss_emp = c.crpss_emp; crps_emp_fair = c.crps_emp_fair; + spread_md = c.spread_md; crps_gaus = c.crps_gaus; crpscl_gaus = c.crpscl_gaus; @@ -246,6 +248,8 @@ void ECNTInfo::set(const PairDataEnsemble &pd) { crps_emp_fair = pd.crps_emp_fair_na.wmean(pd.wgt_na); if(is_eq(crps_emp_fair, 0.0)) crps_emp_fair = 0.0; + + spread_md = pd.spread_md_na.wmean(pd.wgt_na); crpscl_emp = pd.crpscl_emp_na.wmean(pd.wgt_na); crpss_emp = (is_bad_data(crps_emp) || diff --git a/src/libcode/vx_statistics/ens_stats.h b/src/libcode/vx_statistics/ens_stats.h index 08b2a19baa..399df564ed 100644 --- a/src/libcode/vx_statistics/ens_stats.h +++ b/src/libcode/vx_statistics/ens_stats.h @@ -76,7 +76,7 @@ class ECNTInfo { // Number of ensemble members and pairs int n_ens, n_pair; - double crps_emp, crpscl_emp, crpss_emp, crps_emp_fair; + double crps_emp, crpscl_emp, crpss_emp, crps_emp_fair, spread_md; double crps_gaus, crpscl_gaus, crpss_gaus; double ign, me, mae, rmse, spread; double me_oerr, mae_oerr, rmse_oerr, spread_oerr; diff --git a/src/libcode/vx_statistics/pair_data_ensemble.cc b/src/libcode/vx_statistics/pair_data_ensemble.cc index da1345528a..f9aa0ecf08 100644 --- a/src/libcode/vx_statistics/pair_data_ensemble.cc +++ b/src/libcode/vx_statistics/pair_data_ensemble.cc @@ -100,6 +100,7 @@ void PairDataEnsemble::clear() { crps_emp_na.clear(); crps_emp_fair_na.clear(); + spread_md_na.clear(); crpscl_emp_na.clear(); crps_gaus_na.clear(); crpscl_gaus_na.clear(); @@ -172,6 +173,7 @@ void PairDataEnsemble::extend(int n) { r_na.extend (n); crps_emp_na.extend (n); crps_emp_fair_na.extend (n); + spread_md_na.extend (n); crpscl_emp_na.extend (n); crps_gaus_na.extend (n); crpscl_gaus_na.extend (n); @@ -231,15 +233,16 @@ void PairDataEnsemble::assign(const PairDataEnsemble &pd) { cdf_na = pd.cdf_na; // PairDataEnsemble - v_na = pd.v_na; - r_na = pd.r_na; - crps_emp_na = pd.crps_emp_na; + v_na = pd.v_na; + r_na = pd.r_na; + crps_emp_na = pd.crps_emp_na; crps_emp_fair_na = pd.crps_emp_fair_na; - crpscl_emp_na = pd.crpscl_emp_na; - crps_gaus_na = pd.crps_gaus_na; - crpscl_gaus_na = pd.crpscl_gaus_na; - ign_na = pd.ign_na; - pit_na = pd.pit_na; + spread_md_na = pd.spread_md_na; + crpscl_emp_na = pd.crpscl_emp_na; + crps_gaus_na = pd.crps_gaus_na; + crpscl_gaus_na = pd.crpscl_gaus_na; + ign_na = pd.ign_na; + pit_na = pd.pit_na; n_ge_obs_na = pd.n_ge_obs_na; me_ge_obs_na = pd.me_ge_obs_na; @@ -440,6 +443,7 @@ void PairDataEnsemble::compute_pair_vals(const gsl_rng *rng_ptr) { r_na.add(bad_data_int); crps_emp_na.add(bad_data_double); crps_emp_fair_na.add(bad_data_double); + spread_md_na.add(bad_data_double); crpscl_emp_na.add(bad_data_double); crps_gaus_na.add(bad_data_double); crpscl_gaus_na.add(bad_data_double); @@ -507,6 +511,7 @@ void PairDataEnsemble::compute_pair_vals(const gsl_rng *rng_ptr) { double crps_emp = compute_crps_emp(o_na[i], cur_ens); crps_emp_na.add(crps_emp); crps_emp_fair_na.add(crps_emp - cur_ens.wmean_abs_diff()); + spread_md_na.add(cur_ens.mean_abs_diff()); crpscl_emp_na.add(compute_crps_emp(o_na[i], cur_clm)); // Ensemble mean and standard deviation @@ -859,7 +864,7 @@ PairDataEnsemble PairDataEnsemble::subset_pairs_obs_thresh(const SingleThresh &o // // Include in subset: // wgt_na, o_na, cmn_na, csd_na, v_na, r_na, - // crps_emp_na, crps_emp_fair_na, crpscl_emp_na, crps_gaus_na, crpscl_gaus_na, + // crps_emp_na, crps_emp_fair_na, spread_md_na, crpscl_emp_na, crps_gaus_na, crpscl_gaus_na, // ign_na, pit_na, n_gt_obs_na, me_gt_obs_na, n_lt_obs_na, me_lt_obs_na, // var_na, var_oerr_na, var_plus_oerr_na, // mn_na, mn_oerr_na, e_na @@ -877,6 +882,7 @@ PairDataEnsemble PairDataEnsemble::subset_pairs_obs_thresh(const SingleThresh &o pd.r_na.add(r_na[i]); pd.crps_emp_na.add(crps_emp_na[i]); pd.crps_emp_fair_na.add(crps_emp_fair_na[i]); + pd.spread_md_na.add(spread_md_na[i]); pd.crpscl_emp_na.add(crpscl_emp_na[i]); pd.crps_gaus_na.add(crps_gaus_na[i]); pd.crpscl_gaus_na.add(crpscl_gaus_na[i]); diff --git a/src/libcode/vx_statistics/pair_data_ensemble.h b/src/libcode/vx_statistics/pair_data_ensemble.h index 3c68caee7e..5c7d4bb741 100644 --- a/src/libcode/vx_statistics/pair_data_ensemble.h +++ b/src/libcode/vx_statistics/pair_data_ensemble.h @@ -79,7 +79,8 @@ class PairDataEnsemble : public PairBase { NumArray r_na; // Observation ranks [n_obs] NumArray crps_emp_na; // Empirical Continuous Ranked Probability Score [n_obs] - NumArray crps_emp_fair_na; // Empirical Continuous Ranked Probability Score [n_obs], adjusted with the mean absolute difference of the ensmble members + NumArray crps_emp_fair_na; // Fair Empirical Continuous Ranked Probability Score [n_obs] + NumArray spread_md_na; // Mean absolute difference of ensemble members [n_obs] NumArray crpscl_emp_na; // Empirical climatological CRPS [n_obs] NumArray crps_gaus_na; // Gaussian CRPS [n_obs] diff --git a/src/tools/core/stat_analysis/aggr_stat_line.cc b/src/tools/core/stat_analysis/aggr_stat_line.cc index f81edb481f..fe4a55d2ec 100644 --- a/src/tools/core/stat_analysis/aggr_stat_line.cc +++ b/src/tools/core/stat_analysis/aggr_stat_line.cc @@ -2570,7 +2570,7 @@ void aggr_ecnt_lines(LineDataFile &f, STATAnalysisJob &job, AggrENSInfo aggr; ECNTData cur; ConcatString key; - double crps_emp, crps_emp_fair, crpscl_emp, crps_gaus, crpscl_gaus, v; + double crps_emp, crps_emp_fair, spread_md, crpscl_emp, crps_gaus, crpscl_gaus, v; map::iterator it; // @@ -2630,6 +2630,7 @@ void aggr_ecnt_lines(LineDataFile &f, STATAnalysisJob &job, // m[key].ens_pd.crps_emp_na.add(cur.crps_emp); m[key].ens_pd.crps_emp_fair_na.add(cur.crps_emp_fair); + m[key].ens_pd.spread_md_na.add(cur.spread_md); m[key].ens_pd.crpscl_emp_na.add(cur.crpscl_emp); m[key].ens_pd.crps_gaus_na.add(cur.crps_gaus); m[key].ens_pd.crpscl_gaus_na.add(cur.crpscl_gaus); @@ -2687,6 +2688,7 @@ void aggr_ecnt_lines(LineDataFile &f, STATAnalysisJob &job, crps_emp = it->second.ens_pd.crps_emp_na.wmean(it->second.ens_pd.wgt_na); crps_emp_fair = it->second.ens_pd.crps_emp_fair_na.wmean(it->second.ens_pd.wgt_na); + spread_md = it->second.ens_pd.spread_md_na.wmean(it->second.ens_pd.wgt_na); crpscl_emp = it->second.ens_pd.crpscl_emp_na.wmean(it->second.ens_pd.wgt_na); crps_gaus = it->second.ens_pd.crps_gaus_na.wmean(it->second.ens_pd.wgt_na); crpscl_gaus = it->second.ens_pd.crpscl_gaus_na.wmean(it->second.ens_pd.wgt_na); @@ -3197,6 +3199,7 @@ void aggr_orank_lines(LineDataFile &f, STATAnalysisJob &job, double crps_emp = compute_crps_emp(cur.obs, cur.ens_na); m[key].ens_pd.crps_emp_na.add(crps_emp); m[key].ens_pd.crps_emp_fair_na.add(crps_emp - cur.ens_na.wmean_abs_diff()); + m[key].ens_pd.spread_md_na.add(cur.ens_na.mean_abs_diff()); m[key].ens_pd.crpscl_emp_na.add(compute_crps_emp(cur.obs, climo_vals)); // Store Gaussian CRPS stats diff --git a/src/tools/core/stat_analysis/parse_stat_line.cc b/src/tools/core/stat_analysis/parse_stat_line.cc index 921297c2a2..a5410e2a53 100644 --- a/src/tools/core/stat_analysis/parse_stat_line.cc +++ b/src/tools/core/stat_analysis/parse_stat_line.cc @@ -357,10 +357,11 @@ void parse_ecnt_line(STATLine &l, ECNTData &e_data) { e_data.total = atoi(l.get_item("TOTAL")); e_data.n_ens = atof(l.get_item("N_ENS")); - e_data.crps_emp = atof(l.get_item("CRPS_EMP")); - e_data.crps_emp_fair = atof(l.get_item("CRPS_EMP_FAIR")); - e_data.crpscl_emp = atof(l.get_item("CRPSCL_EMP")); - e_data.crpss_emp = atof(l.get_item("CRPSS_EMP")); + e_data.crps_emp = atof(l.get_item("CRPS_EMP")); + e_data.crps_emp_fair = atof(l.get_item("CRPS_EMP_FAIR")); + e_data.spread_md = atof(l.get_item("SPREAD_MD")); + e_data.crpscl_emp = atof(l.get_item("CRPSCL_EMP")); + e_data.crpss_emp = atof(l.get_item("CRPSS_EMP")); e_data.crps_gaus = atof(l.get_item("CRPS")); e_data.crpscl_gaus = atof(l.get_item("CRPSCL")); diff --git a/src/tools/core/stat_analysis/parse_stat_line.h b/src/tools/core/stat_analysis/parse_stat_line.h index 3885538b08..62718981b2 100644 --- a/src/tools/core/stat_analysis/parse_stat_line.h +++ b/src/tools/core/stat_analysis/parse_stat_line.h @@ -61,7 +61,7 @@ struct MPRData { // Ensemble continuous statistics (ECNT) data structure struct ECNTData { int total, n_ens; - double crps_emp, crps_emp_fair, crpscl_emp, crpss_emp; + double crps_emp, crps_emp_fair, spread_md, crpscl_emp, crpss_emp; double crps_gaus, crpscl_gaus, crpss_gaus; double ign, me, mae, rmse, spread; double me_oerr, mae_oerr, rmse_oerr, spread_oerr;