From 8c446ffd61fb79224c753060ada2fa355c678018 Mon Sep 17 00:00:00 2001 From: Seth Linden Date: Wed, 3 Nov 2021 09:36:55 -0600 Subject: [PATCH] Feature 1761 percent thresh (#1956) * Per issue #1761 in set_perc() adding code to get FBIAS numeric value, like 1.0 or 0.9, etc. SL * Per issue #1761: in set_perc(), modified actual percentile calculation at end to use the extracted FBIAS numeric value (float). SL * Per issue #1761: modified the check on the perc_thresh_freq_bias, just has to be > 0 now. SL * Per issue #1761: cleaned up code in set_perc(). SL * Per #1761, updates to Simple_Node::set_perc() to handle variable frequency bias amounts. Changes include: - Reverting the formatting of this back to how it originally was in the develop branch. In general, just match the formatting of the existing file, so as the minimize the number of difference lines. - Add logic to adjust the percentile to be found based on the requested FBIAS value. Multiplying or dividing the percentile by the FBIAS value depends on the inequality type and whether we're bias adjusting the forecast or observation data. - Adjust the log messages slightly. Please be aware that I'm not totally confident in these changes. They warrant much more testing. This logic is very, very confusing. * Per #1761, call compute_percentile() when double-checking the percentile values. * Per #1761, remove unused variable. * Per #1761, add warning for percentiles > 100. * Per #1761. In set_perc(), after testing cleaned up code. SL * Per issue #1761: adding new config file for testing dynamic FBIAS values. SL * Per issue #1761: added new unit test for dynamic FBIAS values when running grid_stat. SL * Per issue #1761, modified FBIAS section to indicated that the user can use dynamic values that are not 1.0. SL * Update met/docs/Users_Guide/config_options.rst Co-authored-by: johnhg * Update met/docs/Users_Guide/config_options.rst Co-authored-by: johnhg * Update met/docs/Users_Guide/config_options.rst Co-authored-by: johnhg * Update test/config/GridStatConfig_fbias_perc_thresh Co-authored-by: johnhg * Update test/config/GridStatConfig_fbias_perc_thresh Co-authored-by: johnhg * Update test/config/GridStatConfig_fbias_perc_thresh Co-authored-by: johnhg * Update test/config/GridStatConfig_fbias_perc_thresh Co-authored-by: johnhg * Per issue #1761, set nc_pairs_flag = FALSE. SL Co-authored-by: Seth Linden Co-authored-by: John Halley Gotway --- met/docs/Users_Guide/config_options.rst | 16 +- met/src/basic/vx_config/config.tab.yy | 2 +- met/src/basic/vx_config/threshold.cc | 99 ++++++--- test/config/GridStatConfig_fbias_perc_thresh | 204 +++++++++++++++++++ test/xml/unit_perc_thresh.xml | 24 +++ 5 files changed, 314 insertions(+), 31 deletions(-) create mode 100644 test/config/GridStatConfig_fbias_perc_thresh diff --git a/met/docs/Users_Guide/config_options.rst b/met/docs/Users_Guide/config_options.rst index 23f67fecda..e6d2349cc4 100644 --- a/met/docs/Users_Guide/config_options.rst +++ b/met/docs/Users_Guide/config_options.rst @@ -93,12 +93,14 @@ The configuration file language supports the following data types: e.g. "5.0" and - "fcst.cat_thresh = ==FBIAS1;", MET applies the >5.0 threshold to the - observations and then chooses a forecast threshold which results in a - frequency bias of 1. + * "==FBIAS" for a user-specified frequency bias value. + e.g. "==FBIAS1" to automatically de-bias the data, "==FBIAS0.9" to select a low-bias threshold, or "==FBIAS1.1" to select a high-bias threshold. + This option must be used in + conjunction with a simple threshold in the other field. For example, + when "obs.cat_thresh = >5.0" and "fcst.cat_thresh = ==FBIAS1;", + MET applies the >5.0 threshold to the observations and then chooses a + forecast threshold which results in a frequency bias of 1. + The frequency bias can be any float value > 0.0. * "CDP" for climatological distribution percentile thresholds. These thresholds require that the climatological mean and standard @@ -119,7 +121,7 @@ The configuration file language supports the following data types: For example, "==CDP25" is automatically expanded to 4 percentile bins: >=CDP0&&=CDP25&&=CDP50&&=CDP75&&<=CDP100 - * When sample percentile thresholds of type SFP, SOP, SCP, or FBIAS1 are + * When sample percentile thresholds of type SFP, SOP, SCP, or FBIAS are requested, MET recomputes the actual percentile that the threshold represents. If the requested percentile and actual percentile differ by more than 5%, a warning message is printed. This may occur when the diff --git a/met/src/basic/vx_config/config.tab.yy b/met/src/basic/vx_config/config.tab.yy index d19715eb35..a89d87c6e8 100644 --- a/met/src/basic/vx_config/config.tab.yy +++ b/met/src/basic/vx_config/config.tab.yy @@ -1661,7 +1661,7 @@ if ( s->PT < 0 || s->PT > 100 ) { } -if ( s->Ptype == perc_thresh_freq_bias && !is_eq(s->PT, 1.0) ) { +if ( s->Ptype == perc_thresh_freq_bias && s->PT <= 0 ) { mlog << Error << "\ndo_simple_perc_thresh() -> " << "unsupported frequency bias percentile threshold!\n\n"; diff --git a/met/src/basic/vx_config/threshold.cc b/met/src/basic/vx_config/threshold.cc index cf3e9dd26a..bf317733ff 100644 --- a/met/src/basic/vx_config/threshold.cc +++ b/met/src/basic/vx_config/threshold.cc @@ -934,10 +934,11 @@ void Simple_Node::set_perc(const NumArray *fptr, const NumArray *optr, const Num { -int i, count; +int i; double ptile, diff; NumArray data; const NumArray * ptr = 0; +bool fbias_fcst = false; // // handle sample percentile types @@ -972,14 +973,16 @@ else if ( Ptype == perc_thresh_freq_bias ) { fthr->get_ptype() == no_perc_thresh_type && fthr->get_type() != thresh_complex ) { + fbias_fcst = false; + ptr = optr; op = fthr->get_type(); PT = fptr->compute_percentile(fthr->get_value(), is_inclusive(fthr->get_type())); mlog << Debug(3) - << "The forecast threshold \"" << fthr->get_str() - << "\" includes " << PT * 100.0 << "% of the data.\n"; + << "The forecast threshold value \"" << fthr->get_str() + << "\" represents the " << PT * 100.0 << "-th percentile.\n"; } @@ -991,14 +994,16 @@ else if ( Ptype == perc_thresh_freq_bias ) { othr->get_ptype() == no_perc_thresh_type && othr->get_type() != thresh_complex ) { + fbias_fcst = true; + ptr = fptr; op = othr->get_type(); PT = optr->compute_percentile(othr->get_value(), is_inclusive(othr->get_type())); mlog << Debug(3) - << "The observation threshold \"" << othr->get_str() - << "\" includes " << PT * 100.0 << "% of the data.\n"; + << "The observation threshold value \"" << othr->get_str() + << "\" represents the " << PT * 100.0 << "-th percentile.\n"; } @@ -1029,7 +1034,8 @@ else if ( Ptype == perc_thresh_freq_bias ) { PT *= 100.0; -} +} // end else if PT == perc_thresh_freq_bias + // // nothing to do // @@ -1082,7 +1088,7 @@ if ( data.n() == 0 ) { << " threshold \"" << s << "\" because no valid data was provided.\n\n"; - exit ( 1 ); + exit ( 1 ); } else { @@ -1094,6 +1100,64 @@ else { s.strip_paren(); abbr_s.strip_paren(); + // + // parse the frequency bias value from the threshold string + // + + if ( Ptype == perc_thresh_freq_bias ) { + + ConcatString fs = s; + + fs.replace("==FBIAS", " ", false); + + double fbias_val = atof(fs.c_str()); + + // + // range check requested bias value + // + + if ( fbias_val <= 0.0 ) { + + mlog << Error << "\nSimple_Node::set_perc() -> " + << "the requested frequency bias value (" << fbias_val + << ") must be > 0 in threshold \"" << s << "\".\n\n"; + + } + + // + // adjust PT by the requested frequency bias amount + // + + double PT_new; + + if ( fbias_fcst ) { + if ( op == thresh_le || op == thresh_lt ) PT_new = PT * fbias_val; + else if ( op == thresh_ge || op == thresh_gt ) PT_new = PT / fbias_val; + } + else { + if ( op == thresh_le || op == thresh_lt ) PT_new = PT / fbias_val; + else if ( op == thresh_ge || op == thresh_gt ) PT_new = PT * fbias_val; + } + + if ( PT_new > 100.0 ) { + mlog << Warning << "\nSimple_Node::set_perc() -> " + << "For " << (fbias_fcst ? "forecast" : "observation" ) + << " threshold \"" << s << "\" the required percentile of " + << PT_new << " exceeds the maximum possible value. " + << "Resetting to 100.\n\n"; + + PT_new = 100.0; + } + + mlog << Debug(3) + << "For " << (fbias_fcst ? "forecast" : "observation" ) + << " threshold \"" << s << "\" with type \"" << thresh_type_str[op] + << "\" update the requested percentile from " << PT << " to " + << PT_new << ".\n"; + + PT = PT_new; + } + // // compute the percentile and update the strings // @@ -1117,27 +1181,16 @@ else { // compute the actual percentile and check tolerance // - if ( op == thresh_le || op == thresh_ge || op == thresh_eq ) { - - for ( i=count=0; i " << "the requested percentile (" << PT - << "%) for threshold \"" << s + << ") for threshold \"" << s << "\" differs from the actual percentile (" - << ptile * 100.0 << ") by " << diff * 100.0 << "%.\n" + << ptile * 100.0 << ") by " << diff * 100.0 << ".\n" << "This is common for small samples or data that contains " << "ties.\n\n"; @@ -1146,8 +1199,8 @@ else { mlog << Debug(3) << "The requested percentile (" << PT - << "%) for threshold threshold \"" << s - << "\" includes " << ptile * 100.0 << "% of the data.\n"; + << ") for threshold \"" << s << "\" includes " + << ptile * 100.0 << "% of the data.\n"; } diff --git a/test/config/GridStatConfig_fbias_perc_thresh b/test/config/GridStatConfig_fbias_perc_thresh new file mode 100644 index 0000000000..1072db748a --- /dev/null +++ b/test/config/GridStatConfig_fbias_perc_thresh @@ -0,0 +1,204 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Grid-Stat configuration file. +// +// For additional information, please see the MET User's Guide. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +model = "GFS"; + +// +// Output description to be written +// May be set separately in each "obs.field" entry +// +desc = "NA"; + +// +// Output observation type to be written +// +obtype = "ANALYS"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = { + to_grid = NONE; + method = NEAREST; + width = 1; + vld_thresh = 0.5; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// May be set separately in each "field" entry +// +censor_thresh = []; +censor_val = []; +mpr_column = []; +mpr_thresh = []; +cat_thresh = []; +cnt_thresh = [ NA ]; +cnt_logic = UNION; +wind_thresh = [ NA ]; +wind_logic = UNION; +eclv_points = 0.05; +nc_pairs_var_name = ""; +nc_pairs_var_suffix = ""; +hss_ec_value = NA; +rank_corr_flag = FALSE; + +// +// Forecast and observation fields to be verified +// +fcst = { + + field = [ + { name = "TMP"; level = [ "P500" ]; cat_thresh = [ ==FBIAS0.9, ==FBIAS1.0, ==FBIAS1.1, ==FBIAS0.9, ==FBIAS1.0, ==FBIAS1.1, <243, <243, <243, >253, >253, >253 ]; } + ]; + +} +obs = { + + field = [ + { name = "TMP"; level = [ "P500" ]; cat_thresh = [ <243, <243, <243, >253, >253, >253, ==FBIAS0.9, ==FBIAS1.0, ==FBIAS1.1, ==FBIAS0.9, ==FBIAS1.0, ==FBIAS1.1 ]; } + ]; + +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +mask = { + grid = [ "FULL" ]; + poly = [ ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Confidence interval settings +// +ci_alpha = [ 0.05 ]; + +boot = { + interval = PCTILE; + rep_prop = 1.0; + n_rep = 0; + rng = "mt19937"; + seed = "1"; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Data smoothing methods +// +interp = { + field = FCST; + vld_thresh = 1.0; + shape = SQUARE; + + type = [ + { method = NEAREST; width = 1; } + ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Neighborhood methods +// +nbrhd = { + width = [ 3, 5 ]; + cov_thresh = [ >=0.5 ]; + vld_thresh = 1.0; + shape = SQUARE; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fourier decomposition +// +fourier = { + wave_1d_beg = []; + wave_1d_end = []; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Gradient statistics +// May be set separately in each "obs.field" entry +// +gradient = { + dx = [ 1 ]; + dy = [ 1 ]; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Distance Map statistics +// May be set separately in each "obs.field" entry +// +distance_map = { + baddeley_p = 2; + baddeley_max_dist = NA; + fom_alpha = 0.1; + zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Statistical output types +// +output_flag = { + fho = NONE; + ctc = BOTH; + cts = BOTH; + mctc = NONE; + mcts = NONE; + cnt = NONE; + sl1l2 = NONE; + sal1l2 = NONE; + vl1l2 = NONE; + val1l2 = NONE; + vcnt = NONE; + pct = NONE; + pstd = NONE; + pjc = NONE; + prc = NONE; + eclv = NONE; + nbrctc = NONE; + nbrcts = NONE; + nbrcnt = NONE; + grad = NONE; + dmap = NONE; +} + +// +// NetCDF matched pairs output file +// +nc_pairs_flag = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +grid_weight_flag = NONE; +tmp_dir = "/tmp"; +output_prefix = "${OUTPUT_PREFIX}"; +version = "V10.1.0"; + +//////////////////////////////////////////////////////////////////////////////// diff --git a/test/xml/unit_perc_thresh.xml b/test/xml/unit_perc_thresh.xml index bc2ef52a83..23e1c83aea 100644 --- a/test/xml/unit_perc_thresh.xml +++ b/test/xml/unit_perc_thresh.xml @@ -84,6 +84,30 @@ + + + + + + &MET_BIN;/grid_stat + + OUTPUT_PREFIX PERC_THRESH_FBIAS + + \ + &DATA_DIR_MODEL;/grib2/gfs/gfs_2012040900_F024.grib2 \ + &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120410_0000_000.grb2 \ + &CONFIG_DIR;/GridStatConfig_fbias_perc_thresh \ + -outdir &OUTPUT_DIR;/perc_thresh -v 3 + + + &OUTPUT_DIR;/perc_thresh/grid_stat_PERC_THRESH_FBIAS_240000L_20120410_000000V.stat + &OUTPUT_DIR;/perc_thresh/grid_stat_PERC_THRESH_FBIAS_240000L_20120410_000000V_cts.txt + &OUTPUT_DIR;/perc_thresh/grid_stat_PERC_THRESH_FBIAS_240000L_20120410_000000V_cnt.txt + &OUTPUT_DIR;/perc_thresh/grid_stat_PERC_THRESH_FBIAS_240000L_20120410_000000V_sl1l2.txt + &OUTPUT_DIR;/perc_thresh/grid_stat_PERC_THRESH_FBIAS_240000L_20120410_000000V_pairs.nc + + +