diff --git a/met/data/config/GridStatConfig_default b/met/data/config/GridStatConfig_default index e79a3e42b1..89dc90e156 100644 --- a/met/data/config/GridStatConfig_default +++ b/met/data/config/GridStatConfig_default @@ -202,6 +202,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/met/data/table_files/met_header_columns_V10.1.txt b/met/data/table_files/met_header_columns_V10.1.txt index d538d74c92..87b5eba258 100644 --- a/met/data/table_files/met_header_columns_V10.1.txt +++ b/met/data/table_files/met_header_columns_V10.1.txt @@ -10,7 +10,7 @@ V10.1 : STAT : NBRCNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID V10.1 : STAT : NBRCTC : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FY_OY FY_ON FN_OY FN_ON V10.1 : STAT : NBRCTS : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL BASER BASER_NCL BASER_NCU BASER_BCL BASER_BCU FMEAN FMEAN_NCL FMEAN_NCU FMEAN_BCL FMEAN_BCU ACC ACC_NCL ACC_NCU ACC_BCL ACC_BCU FBIAS FBIAS_BCL FBIAS_BCU PODY PODY_NCL PODY_NCU PODY_BCL PODY_BCU PODN PODN_NCL PODN_NCU PODN_BCL PODN_BCU POFD POFD_NCL POFD_NCU POFD_BCL POFD_BCU FAR FAR_NCL FAR_NCU FAR_BCL FAR_BCU CSI CSI_NCL CSI_NCU CSI_BCL CSI_BCU GSS GSS_BCL GSS_BCU HK HK_NCL HK_NCU HK_BCL HK_BCU HSS HSS_BCL HSS_BCU ODDS ODDS_NCL ODDS_NCU ODDS_BCL ODDS_BCU LODDS LODDS_NCL LODDS_NCU LODDS_BCL LODDS_BCU ORSS ORSS_NCL ORSS_NCU ORSS_BCL ORSS_BCU EDS EDS_NCL EDS_NCU EDS_BCL EDS_BCU SEDS SEDS_NCL SEDS_NCU SEDS_BCL SEDS_BCU EDI EDI_NCL EDI_NCU EDI_BCL EDI_BCU SEDI SEDI_NCL SEDI_NCU SEDI_BCL SEDI_BCU BAGSS BAGSS_BCL BAGSS_BCU V10.1 : STAT : GRAD : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FGBAR OGBAR MGBAR EGBAR S1 S1_OG FGOG_RATIO DX DY -V10.1 : STAT : DMAP : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FY OY FBIAS BADDELEY HAUSDORFF MED_FO MED_OF MED_MIN MED_MAX MED_MEAN FOM_FO FOM_OF FOM_MIN FOM_MAX FOM_MEAN ZHU_FO ZHU_OF ZHU_MIN ZHU_MAX ZHU_MEAN +V10.1 : STAT : DMAP : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FY OY FBIAS BADDELEY HAUSDORFF MED_FO MED_OF MED_MIN MED_MAX MED_MEAN FOM_FO FOM_OF FOM_MIN FOM_MAX FOM_MEAN ZHU_FO ZHU_OF ZHU_MIN ZHU_MAX ZHU_MEAN G GBETA BETA_VALUE V10.1 : STAT : ORANK : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL INDEX OBS_SID OBS_LAT OBS_LON OBS_LVL OBS_ELV OBS PIT RANK N_ENS_VLD (N_ENS) ENS_[0-9]* OBS_QC ENS_MEAN CLIMO_MEAN SPREAD ENS_MEAN_OERR SPREAD_OERR SPREAD_PLUS_OERR CLIMO_STDEV V10.1 : STAT : PCT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL (N_THRESH) THRESH_[0-9]* OY_[0-9]* ON_[0-9]* V10.1 : STAT : PJC : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL (N_THRESH) THRESH_[0-9]* OY_TP_[0-9]* ON_TP_[0-9]* CALIBRATION_[0-9]* REFINEMENT_[0-9]* LIKELIHOOD_[0-9]* BASER_[0-9]* diff --git a/met/docs/Users_Guide/appendixC.rst b/met/docs/Users_Guide/appendixC.rst index 10eb14f5d8..d6523fb1c2 100644 --- a/met/docs/Users_Guide/appendixC.rst +++ b/met/docs/Users_Guide/appendixC.rst @@ -1134,7 +1134,7 @@ A mathematical metric, :math:`m(A,B)\geq 0`, must have the following three prope The first establishes that a perfect score is zero and that the only way to obtain a perfect score is if the two sets are identical according to the metric. The second requirement ensures that the order by which the two sets are evaluated will not change the result. The third property ensures that if *C* is closer to *A* than *B* is to *A*, then :math:`m(A,C) < m(A,B)`. -It has been argued in :ref:`Gilleland (2019) ` that the second property of symmetry is not necessarily an important quality to have for a summary measure for verification purposes because lack of symmetry allows for information about false alarms and misses. +It has been argued in :ref:`Gilleland (2017) ` that the second property of symmetry is not necessarily an important quality to have for a summary measure for verification purposes because lack of symmetry allows for information about false alarms and misses. The results of the distance map verification approaches that are included in the Grid-Stat tool are summarized using a variety of measures. These measures include Baddeley's :math:`\Delta` Metric, the Hausdorff Distance, the Mean-error Distance, Pratt's Figure of Merit, and Zhu's Measure. Their equations are listed below. @@ -1205,6 +1205,29 @@ where MED *(A,B)* is as in the Mean-error distance, *N* is the total number of g The range for ZHU is 0 to infinity, with a score of 0 indicating a perfect forecast. +.. _App_C-gbeta: + +:math:`G` and :math:`G_\beta` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Called "G" and "GBETA" in the DMAP output :numref:`table_GS_format_info_DMAP` + +See :numref:`grid-stat_gbeta` for a description. + +Let :math:`y = {y_1}{y_2}` where :math:`y_1 = n_A + n_B - 2n_{AB}`, and :math:`y_2 = MED(A,B) \cdot n_B + MED(B,A) \cdot n_A`, with the mean-error distance (:math:`MED`) as described above, and where :math:`n_{A}`, :math:`n_{B}`, and :math:`n_{AB}` are the number of events within event areas *A*, *B*, and the intersection of *A* and *B*, respectively. + +The :math:`G` performance measure is given by + +.. math:: G(A,B) = y^{1/3} + +and the :math:`G_\beta` performance measure is given by + +.. math:: G_\beta(A,B) = max\{1-\frac{y}{\beta}, 0\} + +where :math:`\beta > 0` is a user-chosen parameter with a default value of :math:`n^2 / 2.0` with :math:`n` equal to the number of points in the domain. The square-root of :math:`G` will give units of grid points, where :math:`y^{1/3}` gives units of grid points squared. + +The range for :math:`G_\beta` is 0 to 1, with a score of 1 indicating a perfect forecast. + Calculating Percentiles _______________________ diff --git a/met/docs/Users_Guide/appendixD.rst b/met/docs/Users_Guide/appendixD.rst index a90c71b620..66c578e91a 100644 --- a/met/docs/Users_Guide/appendixD.rst +++ b/met/docs/Users_Guide/appendixD.rst @@ -77,8 +77,9 @@ All other verification scores with CIs in MET must be obtained through bootstrap 5. Calculate CIs for the parameters directly from the sample (see text below for more details) -Typically, a simple random sample is taken for step 2, and that is how it is done in MET. As an example of what happens in this step, suppose our sample is :math:`X_1,X_2,X_3,X_4`. Then, one possible replicate might be :math:`X_2,X_2,X_2,X_4`. Usually one samples :math:`m = n` points in this step, but there are cases where one should use :math:`m < n`. For example, when the underlying distribution is heavy-tailed, one should use a smaller size m than n (e.g., the closest integer value to the square root of the original sample size). +Typically, a simple random sample is taken for step 2, and that is how it is done in MET. As an example of what happens in this step, suppose our sample is :math:`X_1,X_2,X_3,X_4`. Then, one possible replicate might be :math:`X_2,X_2,X_2,X_4`. Usually one samples :math:`m = n` points in this step, but there are cases where one should use :math:`m < n`. For example, when the underlying distribution is heavy-tailed, one should use a smaller size m than n (e.g., the closest integer value to the square root of the original sample size). See :ref:`Gilleland (2020, part II) ` for considerably more information about the issues with estimators that follow a heavy tailed distribution and the closely related issue of bootstrapping extreme-valued estimators, such as the maximum, in the atmospheric science domain. There are numerous ways to construct CIs from the sample obtained in step 4. MET allows for two of these procedures: the percentile and the BCa. The percentile is the most commonly known method, and the simplest to understand. It is merely the :math:`\alpha / 2` and :math:`1 - \alpha / 2` percentiles from the sample of statistics. Unfortunately, however, it has been shown that this interval is too optimistic in practice (i.e., it doesn't have accurate coverage). One solution is to use the BCa method, which is very accurate, but it is also computationally intensive. This method adjusts for bias and non-constant variance, and yields the percentile interval in the event that the sample is unbiased with constant variance. -If there is dependency in the sample, then it is prudent to account for this dependency in some way. One method that does not make a lot of assumptions is circular block bootstrapping. This is not currently implemented in MET, but will be available in a future release. At that time, the method will be explained more fully here, but until then consult :ref:`Gilleland (2010) ` for more details. +If there is dependency in the sample, then it is prudent to account for this dependency in some way. :ref:`Gilleland (2010) ` describes the bootstrap procedure, along with the above-mentioned parametric methods, in more detail specifically for the verification application. If there is dependency in the sample, then it is prudent to account for this dependency in some way (see :ref:`Gilleland (2020, part I) ` part I for an in-depth discussion of bootstrapping in the competing forecast verification domain). One method that is particularly appropriate for serially dependent data is the circular block resampling procedure for step 2. + diff --git a/met/docs/Users_Guide/config_options.rst b/met/docs/Users_Guide/config_options.rst index a490ea1e27..5c3149b03f 100644 --- a/met/docs/Users_Guide/config_options.rst +++ b/met/docs/Users_Guide/config_options.rst @@ -413,7 +413,7 @@ e.g. model = "GFS"; model = "WRF"; -.._desc: +.. _desc: :ref:`desc ` @@ -1615,136 +1615,6 @@ This dictionary may include the following entries: ]; } -.. _nbrhd: - -:ref:`nbrhd ` - -The "nbrhd" entry is a dictionary that is very similar to the "interp" -entry. It specifies information for computing neighborhood statistics in -Grid-Stat. This dictionary may include the following entries: - -* The "field" entry specifies to which field(s) the computation of - fractional coverage should be applied. Grid-Stat processes each - combination of categorical threshold and neighborhood width to - derive the fractional coverage fields from which neighborhood - statistics are calculated. Users who have computed fractional - coverage fields outside of MET can use this option to disable - these computations. Instead, the raw input values will be - used directly to compute neighborhood statistics: - - * "BOTH" to compute fractional coverage for both the forecast - and the observation fields (default). - - * "FCST" to only process the forecast field. - - * "OBS" to only process the observation field. - - * "NONE" to process neither field. - -* The "vld_thresh" entry is described above. - -* The "shape" entry defines the shape of the neighborhood. - Valid values are "SQUARE" or "CIRCLE" - -* The "width" entry is as described above, and must be odd. - -* The "cov_thresh" entry is an array of thresholds to be used when - computing categorical statistics for the neighborhood fractional - coverage field. - -.. code-block:: none - - nbrhd = { - field = BOTH; - vld_thresh = 1.0; - shape = SQUARE; - width = [ 1 ]; - cov_thresh = [ >=0.5 ]; - } - -.. _fourier: - -:ref:`fourier ` - -The "fourier" entry is a dictionary which specifies the application of the -Fourier decomposition method. It consists of two arrays of the same length -which define the beginning and ending wave numbers to be included. If the -arrays have length zero, no Fourier decomposition is applied. For each array -entry, the requested Fourier decomposition is applied to the forecast and -observation fields. The beginning and ending wave numbers are indicated in -the MET ASCII output files by the INTERP_MTHD column (e.g. WV1_0-3 for waves -0 to 3 or WV1_10 for only wave 10). This 1-dimensional Fourier decomposition -is computed along the Y-dimension only (i.e. the columns of data). It is only -defined when each grid point contains valid data. If either input field -contains missing data, no Fourier decomposition is computed. - -The available wave numbers start at 0 (the mean across each row of data) -and end at (Nx+1)/2 (the finest level of detail), where Nx is the X-dimension -of the verification grid: - -* The "wave_1d_beg" entry is an array of integers specifying the first - wave number to be included. - -* The "wave_1d_end" entry is an array of integers specifying the last - wave number to be included. - -.. code-block:: none - - fourier = { - wave_1d_beg = [ 0, 4, 10 ]; - wave_1d_end = [ 3, 9, 20 ]; - } - -.. _gradient: - -:ref:`gradient ` - -The "gradient" entry is a dictionary which specifies the number and size of -gradients to be computed. The "dx" and "dy" entries specify the size of the -gradients in grid units in the X and Y dimensions, respectively. dx and dy -are arrays of integers (positive or negative) which must have the same -length, and the GRAD output line type will be computed separately for each -entry. When computing gradients, the value at the (x, y) grid point is -replaced by the value at the (x+dx, y+dy) grid point minus the value at -(x, y). - -This configuration option may be set separately in each "obs.field" entry. - -.. code-block:: none - - gradient = { - dx = [ 1 ]; - dy = [ 1 ]; - } - -.. _distance_map: - -:ref:`distance_map ` - -The "distance_map" entry is a dictionary containing options related to the -distance map statistics in the DMAP output line type. The "baddeley_p" entry -is an integer specifying the exponent used in the Lp-norm when computing the -Baddeley Delta metric. The "baddeley_max_dist" entry is a floating point -number specifying the maximum allowable distance for each distance map. Any -distances larger than this number will be reset to this constant. A value of -NA indicates that no maximum distance value should be used. The "fom_alpha" -entry is a floating point number specifying the scaling constant to be used -when computing Pratt's Figure of Merit. The "zhu_weight" specifies a value -between 0 and 1 to define the importance of the RMSE of the binary fields -(i.e. amount of overlap) versus the mean-error distance (MED). The default -value of 0.5 gives equal weighting. - -This configuration option may be set separately in each "obs.field" entry. - -.. code-block:: none - - distance_map = { - baddeley_p = 2; - baddeley_max_dist = NA; - fom_alpha = 0.1; - zhu_weight = 0.5; - } - .. _land_mask: :ref:`land_mask ` diff --git a/met/docs/Users_Guide/figure/grid-stat_fig6.png b/met/docs/Users_Guide/figure/grid-stat_fig6.png new file mode 100644 index 0000000000..e0916152f7 Binary files /dev/null and b/met/docs/Users_Guide/figure/grid-stat_fig6.png differ diff --git a/met/docs/Users_Guide/grid-stat.rst b/met/docs/Users_Guide/grid-stat.rst index f626f77ef8..264903825f 100644 --- a/met/docs/Users_Guide/grid-stat.rst +++ b/met/docs/Users_Guide/grid-stat.rst @@ -120,7 +120,30 @@ While :numref:`grid-stat_fig1` and :numref:`grid-stat_fig2` are helpful in illus The absolute difference between the distance maps in the bottom row of :numref:`grid-stat_fig3` (top left), the shortest distances from every grid point in B to the nearest grid point in A (top right), and the shortest distances from every grid point in A to the nearest grid points in B (bottom left). The latter two do not have axes in order to emphasize that the distances are now only considered from within the respective event sets. The top right graphic is the distance map of A conditioned on the presence of an event from B, and that in the bottom left is the distance map of B conditioned on the presence of an event from A. -The statistics derived from these distance maps are described in :numref:`Appendix C, Section %s `. For each combination of input field and categorical threshold requested in the configuration file, Grid-Stat applies that threshold to define events in the forecast and observation fields and computes distance maps for those binary fields. Statistics for all requested masking regions are derived from those distance maps. Note that the distance maps are computed only once over the full verification domain, not separately for each masking region. Events occurring outside of a masking region can affect the distance map values inside that masking region and, therefore, can also affect the distance maps statistics for that region. +The statistics derived from these distance maps are described in :numref:`Appendix C, Section %s `. To make fair comparisons, any grid point containing bad data in either the forecast or observation field is set to bad data in both fields. For each combination of input field and categorical threshold requested in the configuration file, Grid-Stat applies that threshold to define events in the forecast and observation fields and computes distance maps for those binary fields. Statistics for all requested masking regions are derived from those distance maps. Note that the distance maps are computed only once over the full verification domain, not separately for each masking region. Events occurring outside of a masking region can affect the distance map values inside that masking region and, therefore, can also affect the distance maps statistics for that region. + +.. _grid-stat_gbeta: + +:math:`\beta` and :math:`G_\beta` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +See :numref:`App_C-gbeta` for the :math:`G` and :math:`G_\beta` equations. + +:math:`G_\beta` provides a summary measure of forecast quality for each user-defined threshold chosen. It falls into a range from zero to one where one is a perfect forecast and zero is considered to be a very poor forecast as determined by the user through the value of :math:`\beta`. Values of :math:`G_\beta` closer to one represent better forecasts and worse forecasts as it decreases toward zero. Although a particular value cannot be universally compared against any forecast, when applied with the same choice of :math:`\beta` for the same variable and on the same domain, it is highly effective at ranking such forecasts. + +:math:`G_\beta` is sensitive to the choice of :math:`\beta`, which depends on the (i) specific domain, (ii) variable, and (iii) user’s needs. Smaller values make :math:`G_\beta` more stringent and larger values make it more lenient. :numref:`grid-stat_fig6` shows an example of applying :math:`G_\beta` over a range of :math:`\beta` values to a precipitation verification set where the binary fields are created by applying a threshold of :math:`2.1 mmh^{-1}`. Color choice and human bias can make it difficult to determine the quality of the forecast for a human observer looking at the raw images in the top row of the figure (:ref:`Ahijevych et al., 2009 `). The bottom left panel of the figure displays the differences in their binary fields, which highlights that the forecast captured the overall shapes of the observed rain areas but suffers from a spatial displacement error (perhaps really a timing error). + +Whether or not the forecast from :numref:`grid-stat_fig6` is “good” or not depends on the specific user. Is it sufficient that the forecast came as close as it did to the observation field? If the answer is yes for the user, then a higher choice of :math:`\beta`, such as :math:`N/2`, with :math:`N` equal to the number of points in the domain, will correctly inform this user that it is a “good” forecast as it will lead to a :math:`G_\beta` value near one. If the user requires the forecast to be much better aligned spatially with the observation field, then a lower choice, perhaps :math:`\beta = N`, will correctly inform that the forecast suffers from spatial displacement errors that are too large for this user to be pleased. If the goal is to rank a series of ensemble forecasts, for example, then a choice of :math:`\beta` that falls in the steep part of the curve shown in the lower right panel of the figure should be preferred, say somewhere between :math:`\beta = N` and :math:`\beta = N^2/2`. Such a choice will ensure that each member is differentiated by the measure. + +.. _grid-stat_fig6: + +.. figure:: figure/grid-stat_fig6.png + + Top left is an example of an accumulated precipitation (mm/h) forecast with the corresponding observed field on the top right. Bottom left shows the difference in binary fields, where the binary fields are created by setting all values in the original fields that fall above :math:`2.1 mmh^{-1}` to one and the rest to zero. Bottom right shows the results for :math:`G_\beta` calculated on the binary fields using the threshold of :math:`2.1 mmh^{-1}` over a range of choices for :math:`\beta`. + +In some cases, a user may be interested in a much higher threshold than :math:`2.1 mmh^{-1}` of the above example. :ref:`Gilleland, 2021 (Fig. 4) `, for example, shows this same forecast using a threshold of :math:`40 mmh^{-1}`. Only a small area in Mississippi has such extreme rain predicted at this valid time; yet none was observed. Small spatial areas of extreme rain in the observed field, however, did occur in a location far away from Mississippi that was not predicted. Generally, for this type of verification, the Hausdorff metric is a good choice of measure. However, a small choice of :math:`\beta` will provide similar results as the Hausdorff distance (:ref:`Gilleland, 2021 `). The user should think about the average size of storm areas and multiply this value by the displacement distance they are comfortable with in order to get a good initial choice for :math:`\beta`, and may have to increase or decrease its value by trial-and-error using one or two example cases from their verification set. + +Since :math:`G_\beta` is so sensitive to the choice of :math:`\beta`, which is defined relative to the number of points in the verification domain, :math:`G_\beta` is only computed for the full verification domain. :math:`G_\beta` is reported as a bad data value for any masking region subsets of the full verification domain. Practical information _____________________ @@ -234,6 +257,10 @@ The configuration options listed above are common to multiple MET tools and are ___________________________ +.. _nbrhd: + +:ref:`nbrhd ` + .. code-block:: none nbrhd = { @@ -244,7 +271,7 @@ ___________________________ cov_thresh = [ >=0.5 ]; } - + The **nbrhd** dictionary contains a list of values to be used in defining the neighborhood to be used when computing neighborhood verification statistics. The neighborhood **shape** is a **SQUARE** or **CIRCLE** centered on the current point, and the **width** value specifies the width of the square or diameter of the circle as an odd integer. The **field** entry is set to **BOTH, FCST, OBS**, or **NONE** to indicate the fields to which the fractional coverage derivation logic should be applied. This should always be set to **BOTH** unless you have already computed the fractional coverage field(s) with numbers between 0 and 1 outside of MET. @@ -255,6 +282,10 @@ The **cov_thresh** entry contains a comma separated list of thresholds to be app ___________________ +.. _fourier: + +:ref:`fourier ` + .. code-block:: none fourier = { @@ -263,25 +294,36 @@ ___________________ } -The **fourier** entry is a dictionary which specifies the application of the Fourier decomposition method. It consists of two arrays of the same length which define the beginning and ending wave numbers to be included. If the arrays have length zero, no Fourier decomposition is applied. For each array entry, the requested Fourier decomposition is applied to the forecast and observation fields. The beginning and ending wave numbers are indicated in the MET ASCII output files by the INTERP_MTHD column (e.g. WV1_0-3 for waves 0 to 3 or WV1_10 for only wave 10). This 1-dimensional Fourier decomposition is computed along the Y-dimension only (i.e. the columns of data). It is applied to the forecast and observation fields as well as the climatological mean field, if specified. It is only defined when each grid point contains valid data. If any input field contains missing data, no Fourier decomposition is computed. The available wave numbers start at 0 (the mean across each row of data) and end at (Nx+1)/2 (the finest level of detail), where Nx is the X-dimension of the verification grid. +The **fourier** entry is a dictionary which specifies the application of the Fourier decomposition method. It consists of two arrays of the same length which define the beginning and ending wave numbers to be included. If the arrays have length zero, no Fourier decomposition is applied. For each array entry, the requested Fourier decomposition is applied to the forecast and observation fields. The beginning and ending wave numbers are indicated in the MET ASCII output files by the INTERP_MTHD column (e.g. WV1_0-3 for waves 0 to 3 or WV1_10 for only wave 10). This 1-dimensional Fourier decomposition is computed along the Y-dimension only (i.e. the columns of data). It is applied to the forecast and observation fields as well as the climatological mean field, if specified. It is only defined when each grid point contains valid data. If any input field contains missing data, no Fourier decomposition is computed. + +The available wave numbers start at 0 (the mean across each row of data) and end at (Nx+1)/2 (the finest level of detail), where Nx is the X-dimension of the verification grid: + +* The **wave_1d_beg** entry is an array of integers specifying the first wave number to be included. -The **wave_1d_beg** entry is an array of integers specifying the first wave number to be included. The **wave_1d_end** entry is an array of integers specifying the last wave number to be included. +* The **wave_1d_end** entry is an array of integers specifying the last wave number to be included. _____________________ +.. _gradient: + +:ref:`gradient ` + .. code-block:: none - grad = { + gradient = { dx = [ 1 ]; dy = [ 1 ]; } - The **gradient** entry is a dictionary which specifies the number and size of gradients to be computed. The **dx** and **dy** entries specify the size of the gradients in grid units in the X and Y dimensions, respectively. **dx** and **dy** are arrays of integers (positive or negative) which must have the same length, and the GRAD output line type will be computed separately for each entry. When computing gradients, the value at the (x, y) grid point is replaced by the value at the (x+dx, y+dy) grid point minus the value at (x, y). This configuration option may be set separately in each **obs.field** entry. ____________________ +.. _distance_map: + +:ref:`distance_map ` + .. code-block:: none distance_map = { @@ -289,9 +331,10 @@ ____________________ baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } -The **distance_map** entry is a dictionary containing options related to the distance map statistics in the **DMAP** output line type. The **baddeley_p** entry is an integer specifying the exponent used in the Lp-norm when computing the Baddeley Delta metric. The **baddeley_max_dist** entry is a floating point number specifying the maximum allowable distance for each distance map. Any distances larger than this number will be reset to this constant. A value of **NA** indicates that no maximum distance value should be used. The **fom_alpha** entry is a floating point number specifying the scaling constant to be used when computing Pratt's Figure of Merit. The **zhu_weight** specifies a value between 0 and 1 to define the importance of the RMSE of the binary fields (i.e. amount of overlap) versus the mean-error distance (MED). The default value of 0.5 gives equal weighting. This configuration option may be set separately in each **obs.field** entry. +The **distance_map** entry is a dictionary containing options related to the distance map statistics in the **DMAP** output line type. The **baddeley_p** entry is an integer specifying the exponent used in the Lp-norm when computing the Baddeley :math:`\Delta` metric. The **baddeley_max_dist** entry is a floating point number specifying the maximum allowable distance for each distance map. Any distances larger than this number will be reset to this constant. A value of **NA** indicates that no maximum distance value should be used. The **fom_alpha** entry is a floating point number specifying the scaling constant to be used when computing Pratt's Figure of Merit. The **zhu_weight** specifies a value between 0 and 1 to define the importance of the RMSE of the binary fields (i.e. amount of overlap) versus the mean-error distance (MED). The default value of 0.5 gives equal weighting. This configuration option may be set separately in each **obs.field** entry. The **beta_value** entry is defined as a function of n, where n is the total number of grid points in the full verification domain containing valid data in both the forecast and observation fields. The resulting beta_value is used to compute the :math:`G_\beta` statistic. The default function, :math:`N^2 / 2`, is recommended in :ref:`Gilleland, 2021 ` but can be modified as needed. _____________________ @@ -755,7 +798,7 @@ The format of the STAT and ASCII output of the Grid-Stat tool are the same as th - Frequency Bias * - 29 - BADDELEY - - Baddeley's Delta Metric + - Baddeley's :math:`\Delta` Metric * - 30 - HAUSDORFF - Hausdorff Distance @@ -804,6 +847,15 @@ The format of the STAT and ASCII output of the Grid-Stat tool are the same as th * - 45 - ZHU_MEAN - Mean of ZHU_FO and ZHU_OF + * - 46 + - G + - :math:`G` distance measure + * - 47 + - GBETA + - :math:`G_\beta` distance measure + * - 48 + - BETA_VALUE + - Beta value used to compute :math:`G_\beta` If requested using the **nc_pairs_flag** dictionary in the configuration file, a NetCDF file containing the matched pair and forecast minus observation difference fields for each combination of variable type/level and masking region applied will be generated. The contents of this file are determined by the contents of the nc_pairs_flag dictionary. The output NetCDF file is named similarly to the other output files: **grid_stat_PREFIX_ HHMMSSL_YYYYMMDD_HHMMSSV_pairs.nc**. Commonly available NetCDF utilities such as ncdump or ncview may be used to view the contents of the output file. diff --git a/met/docs/Users_Guide/refs.rst b/met/docs/Users_Guide/refs.rst index 090c16ce0f..647e3591bf 100644 --- a/met/docs/Users_Guide/refs.rst +++ b/met/docs/Users_Guide/refs.rst @@ -9,6 +9,13 @@ References | Atlantic Basin. *Weather & Forecasting*, 13, 1005-1015. | +.. _Ahijevych-2009: + +| Ahijevych, D., E. Gilleland, B.G. Brown, and E.E. Ebert, 2009. Application of +| spatial verification methods to idealized and NWP-gridded precipitation forecasts. +| Weather Forecast., 24 (6), 1485 - 1497, doi: 10.1175/2009WAF2222298.1. +| + .. _Barker-1991: @@ -115,12 +122,36 @@ References | Gilleland, E., 2010: Confidence intervals for forecast verification. | *NCAR Technical Note* NCAR/TN-479+STR, 71pp. +| + +.. _Gilleland-2017: + +| Gilleland, E., 2017. A new characterization in the spatial verification +| framework for false alarms, misses, and overall patterns. +| Weather Forecast., 32 (1), 187 - 198, doi: 10.1175/WAF-D-16-0134.1. +| + + +.. _Gilleland_PartI-2020: + +| Gilleland, E., 2020. Bootstrap methods for statistical inference. +| Part I: Comparative forecast verification for continuous variables. +| Journal of Atmospheric and Oceanic Technology, 37 (11), 2117 - 2134, +| doi: 10.1175/JTECH-D-20-0069.1. +| + +.. _Gilleland_PartII-2020: + +| Gilleland, E., 2020. Bootstrap methods for statistical inference. +| Part II: Extreme-value analysis. Journal of Atmospheric and Oceanic +| Technology, 37 (11), 2135 - 2144, doi: 10.1175/JTECH-D-20-0070.1. +| -.. _Gilleland-2019: +.. _Gilleland-2021: -| Gilleland, E., 2019: Bootstrap methods for statistical inference. Part II: -| Extreme-value analysis. Submitted to the Journal of Atmospheric and -| Oceanic Technology on 2 December 2019. Re-submitted on 12 May 2020 +| Gilleland, E., 2021. Novel measures for summarizing high-resolution forecast +| performance. Advances in Statistical Climatology, Meteorology and Oceanography, +| 7 (1), 13 - 34, doi: 10.5194/ascmo-7-13-2021. | .. _Gneiting-2004: diff --git a/met/scripts/config/GridStatConfig_APCP_12 b/met/scripts/config/GridStatConfig_APCP_12 index e63c3a3d23..76747e7908 100644 --- a/met/scripts/config/GridStatConfig_APCP_12 +++ b/met/scripts/config/GridStatConfig_APCP_12 @@ -158,6 +158,8 @@ distance_map = { baddeley_p = 2; baddeley_max_dist = NA; fom_alpha = 0.1; + zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/met/scripts/config/GridStatConfig_APCP_24 b/met/scripts/config/GridStatConfig_APCP_24 index 9ad9e8bb53..bca332cdde 100644 --- a/met/scripts/config/GridStatConfig_APCP_24 +++ b/met/scripts/config/GridStatConfig_APCP_24 @@ -168,6 +168,8 @@ distance_map = { baddeley_p = 2; baddeley_max_dist = NA; fom_alpha = 0.1; + zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/met/scripts/config/GridStatConfig_POP_12 b/met/scripts/config/GridStatConfig_POP_12 index 20c48f59a6..2df3bc3914 100644 --- a/met/scripts/config/GridStatConfig_POP_12 +++ b/met/scripts/config/GridStatConfig_POP_12 @@ -168,6 +168,8 @@ distance_map = { baddeley_p = 2; baddeley_max_dist = NA; fom_alpha = 0.1; + zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/met/scripts/config/GridStatConfig_all b/met/scripts/config/GridStatConfig_all index 901ec29565..13b7f601f3 100644 --- a/met/scripts/config/GridStatConfig_all +++ b/met/scripts/config/GridStatConfig_all @@ -118,7 +118,7 @@ climo_mean = { // Verification masking regions // mask = { - grid = [ "DTC165", "DTC166" ]; + grid = [ "FULL", "DTC165", "DTC166" ]; poly = [ "${TEST_OUT_DIR}/gen_vx_mask/CONUS_poly.nc", "MET_BASE/poly/LMV.poly" ]; } @@ -199,6 +199,8 @@ distance_map = { baddeley_p = 2; baddeley_max_dist = NA; fom_alpha = 0.1; + zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h index d3b8773e58..34fe74c978 100644 --- a/met/src/basic/vx_config/config_constants.h +++ b/met/src/basic/vx_config/config_constants.h @@ -692,6 +692,7 @@ static const char conf_key_baddeley_p[] = "baddeley_p"; static const char conf_key_baddeley_max_dist[] = "baddeley_max_dist"; static const char conf_key_fom_alpha[] = "fom_alpha"; static const char conf_key_zhu_weight[] = "zhu_weight"; +static const char conf_key_beta_value[] = "beta_value"; // // Wavelet-Stat specific parameter key names diff --git a/met/src/basic/vx_util/data_plane.cc b/met/src/basic/vx_util/data_plane.cc index c855b2a1d5..c49c360918 100644 --- a/met/src/basic/vx_util/data_plane.cc +++ b/met/src/basic/vx_util/data_plane.cc @@ -297,6 +297,22 @@ bool DataPlane::is_all_bad_data() const { /////////////////////////////////////////////////////////////////////////////// +int DataPlane::n_good_data() const { + int j, n; + + // + // Count number of good data values + // + + for(j=0,n=0; j " + << "the beta_value (" << beta_value + << ") must be greater than 0!\n\n"; + exit(1); + } + // Initialize clear(); @@ -2852,28 +2868,29 @@ void DMAPInfo::set(const SingleThresh &fthr, const SingleThresh &othr, non_zero_count = 0; - mlog << Debug(4) << " DMAP.Options: baddeley_p=" << baddeley_p - << ", baddeley_max_dist=" << baddeley_max_dist - << ", fom_alpha=" << fom_alpha - << ", zhu_weight=" << zhu_weight << "\n"; - for (int i=0; i 0) { fy++; med_of_sum += odmap_na[i]; fom_of_sum += 1 / (1 + odmap_na[i] * odmap_na[i] * fom_alpha); } + + // Observation if (othr_na[i] > 0) { oy++; med_fo_sum += fdmap_na[i]; fom_fo_sum += 1 / (1 + fdmap_na[i] * fdmap_na[i] * fom_alpha); } + // Forecast and observation event + if (fthr_na[i] > 0 && othr_na[i] > 0) foy++; + sum_event_diff += (fthr_na[i] - othr_na[i]) * (fthr_na[i] - othr_na[i]); f_distance = (!is_bad_data(baddeley_max_dist) && @@ -2937,26 +2954,67 @@ void DMAPInfo::set(const SingleThresh &fthr, const SingleThresh &othr, zhu_mean = (zhu_fo + zhu_of) / 2; } - mlog << Debug(4) << " DMAP: nf=" << fy << ", no=" << oy << ", total=" << total - << "\tbaddeley=" << baddeley << ", hausdorff=" << hausdorff - << "\n\tmed_fo=" << med_fo << ", med_of=" << med_of - << ", med_min=" << med_min << ", med_max=" << med_max << ", med_mean=" << med_mean - << "\n\tfom_fo=" << fom_fo << ", fom_of=" << fom_of - << ", fom_min=" << fom_min << ", fom_max=" << fom_max << ", fom_mean=" << fom_mean - << "\n\tzhu_fo=" << zhu_fo << ", zhu_of=" << zhu_of - << ", zhu_min=" << zhu_min << ", zhu_max=" << zhu_max << ", zhu_mean=" << zhu_mean + // G and G-Beta + // Reference: + // Gilleland, E.: Novel measures for summarizing high-resolution forecast performance, + // Adv. Stat. Clim. Meteorol. Oceanogr., 7, 13–34, + // https://doi.org/10.5194/ascmo-7-13-2021, 2021. + + // If not set by the user, default maximum distance to the number of pairs + double max_dist = (is_bad_data(baddeley_max_dist) ? + (double) total : baddeley_max_dist); + + double g_med_fo = (oy == 0 ? max_dist : med_fo); + double g_med_of = (fy == 0 ? max_dist : med_of); + int g_y1 = fy + oy - 2 * foy; + double g_y2 = g_med_fo * oy + g_med_of * fy; + double g_y = g_y1 * g_y2; + g = pow(g_y, 1.0 / 3.0); + + // Only compute GBETA over the full verification domain. + // Report bad data for masking regions. + if(total == n_full_points) { + gbeta = max(1.0 - g_y / beta_value, 0.0); + } + else { + gbeta = beta_value = bad_data_double; + } + + // Dump debug distance map info + mlog << Debug(4) << " DMAP.Options: baddeley_p=" << baddeley_p + << ", baddeley_max_dist=" << baddeley_max_dist + << ", fom_alpha=" << fom_alpha + << ", zhu_weight=" << zhu_weight + << ", beta_value=" << beta_value + << ", n_full_points=" << n_full_points << "\n"; + + mlog << Debug(4) << " DMAP: nf=" << fy << ", no=" << oy << ", nfo=" << foy << ", total=" << total + << "\n\tbaddeley=" << baddeley << ", hausdorff=" << hausdorff + << "\n\tmed_fo=" << med_fo << ", med_of=" << med_of + << ", med_min=" << med_min << ", med_max=" << med_max << ", med_mean=" << med_mean + << "\n\tfom_fo=" << fom_fo << ", fom_of=" << fom_of + << ", fom_min=" << fom_min << ", fom_max=" << fom_max << ", fom_mean=" << fom_mean + << "\n\tzhu_fo=" << zhu_fo << ", zhu_of=" << zhu_of + << ", zhu_min=" << zhu_min << ", zhu_max=" << zhu_max << ", zhu_mean=" << zhu_mean + << "\n\ty1=" << g_y1 << ", y2=" << g_y2 << ", y=" << g_y + << "\n\tg=" << g << ", gbeta=" << gbeta + << "\n"; + return; } //////////////////////////////////////////////////////////////////////// void DMAPInfo::set_options(const int _baddeley_p, const double _baddeley_max_dist, - const double _fom_alpha, const double _zhu_weight) { + const double _fom_alpha, const double _zhu_weight, + const double _beta_value, const int _n_full_points) { baddeley_p = _baddeley_p; baddeley_max_dist = _baddeley_max_dist; fom_alpha = _fom_alpha; zhu_weight = _zhu_weight; + beta_value = _beta_value; + n_full_points = _n_full_points; } //////////////////////////////////////////////////////////////////////// diff --git a/met/src/libcode/vx_statistics/met_stats.h b/met/src/libcode/vx_statistics/met_stats.h index 9f29299333..06b6af3fcd 100644 --- a/met/src/libcode/vx_statistics/met_stats.h +++ b/met/src/libcode/vx_statistics/met_stats.h @@ -609,6 +609,8 @@ class DMAPInfo { double baddeley_max_dist; // Maximum distance constant double fom_alpha; // FOM Alpha double zhu_weight; // Zhu Weight + double beta_value; // G-Beta Value + int n_full_points; // Number of FULL domain points public: @@ -622,7 +624,7 @@ class DMAPInfo { SingleThresh othresh; // Counts - int total, fy, oy; + int total, fy, oy, foy; // Distance metrics double baddeley, hausdorff; @@ -636,6 +638,9 @@ class DMAPInfo { // Zhu Metric double zhu_fo, zhu_of, zhu_min, zhu_max, zhu_mean; + // G and G-Beta + double g, gbeta; + // Compute statistics double fbias() const; // fbias = fy / oy @@ -645,12 +650,20 @@ class DMAPInfo { const NumArray &fthr_na, const NumArray &othr_na); void set_options(const int _baddeley_p, const double _baddeley_max_dist, - const double _fom_alpha, const double _zhu_weight); + const double _fom_alpha, const double _zhu_weight, + const double _beta_value, const int _n_full_points); + + // Get functions + double get_beta_value() const; void clear(); void reset_options(); }; +//////////////////////////////////////////////////////////////////////// + +inline double DMAPInfo::get_beta_value() const { return(beta_value); } + //////////////////////////////////////////////////////////////////////// // // Utility functions for parsing data from configuration files diff --git a/met/src/tools/core/grid_stat/grid_stat.cc b/met/src/tools/core/grid_stat/grid_stat.cc index f0daf3f6a9..242a3de5e3 100644 --- a/met/src/tools/core/grid_stat/grid_stat.cc +++ b/met/src/tools/core/grid_stat/grid_stat.cc @@ -1210,6 +1210,13 @@ void process_scores() { DataPlane fcst_dp_dmap, obs_dp_dmap; pd.extend(grid.nx()*grid.ny()); + // Mask out missing data between the fields for a fair comparison + DataPlane fcst_dp_mm = fcst_dp; + DataPlane obs_dp_mm = obs_dp; + mask_bad_data(fcst_dp_mm, obs_dp_mm); + mask_bad_data(obs_dp_mm, fcst_dp_mm); + int n_good_data = obs_dp_mm.n_good_data(); + // Loop over the categorical thresholds for(k=0; klookup(conf_key_beta_value)); + if(!beta_value_fx.is_set()) { + mlog << Error << "\nGridStatVxOpt::process_config() -> " + << "The \"" << conf_key_beta_value + << "\" function is not set!\n\n"; + exit(1); + } + // Conf: hss_ec_value hss_ec_value = odict.lookup_double(conf_key_hss_ec_value); diff --git a/met/src/tools/core/grid_stat/grid_stat_conf_info.h b/met/src/tools/core/grid_stat/grid_stat_conf_info.h index ff731985ac..73bc476124 100644 --- a/met/src/tools/core/grid_stat/grid_stat_conf_info.h +++ b/met/src/tools/core/grid_stat/grid_stat_conf_info.h @@ -186,7 +186,8 @@ class GridStatVxOpt { int baddeley_p; // Exponent for lp-norm double baddeley_max_dist; // Maximum distance constant double fom_alpha; // FOM Alpha - double zhu_weight; // Zhu Weight + double zhu_weight; // Zhu Weight + UserFunc_1Arg beta_value_fx; // G-Beta Value Function double hss_ec_value; // MCTS HSS expected correct value bool rank_corr_flag; // Flag for computing rank correlations diff --git a/test/config/GridStatConfig_APCP_regrid b/test/config/GridStatConfig_APCP_regrid index dfcfbf73a7..af00db9f28 100644 --- a/test/config/GridStatConfig_APCP_regrid +++ b/test/config/GridStatConfig_APCP_regrid @@ -150,6 +150,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_GRIB_lvl_typ_val b/test/config/GridStatConfig_GRIB_lvl_typ_val index 1b3a539f57..8ace03905d 100644 --- a/test/config/GridStatConfig_GRIB_lvl_typ_val +++ b/test/config/GridStatConfig_GRIB_lvl_typ_val @@ -247,6 +247,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_GRIB_set_attr b/test/config/GridStatConfig_GRIB_set_attr index dae59b6282..917a0caf66 100644 --- a/test/config/GridStatConfig_GRIB_set_attr +++ b/test/config/GridStatConfig_GRIB_set_attr @@ -179,6 +179,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_GTG_latlon b/test/config/GridStatConfig_GTG_latlon index 0f18b04d2d..36356d9b48 100644 --- a/test/config/GridStatConfig_GTG_latlon +++ b/test/config/GridStatConfig_GTG_latlon @@ -158,6 +158,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_GTG_lc b/test/config/GridStatConfig_GTG_lc index 531a3bb9cf..c32de7a1b7 100644 --- a/test/config/GridStatConfig_GTG_lc +++ b/test/config/GridStatConfig_GTG_lc @@ -158,6 +158,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_apply_mask b/test/config/GridStatConfig_apply_mask index 1c62a9a929..9142054e30 100644 --- a/test/config/GridStatConfig_apply_mask +++ b/test/config/GridStatConfig_apply_mask @@ -159,6 +159,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_climo_WMO b/test/config/GridStatConfig_climo_WMO index 39e64895ce..e23f74a2be 100644 --- a/test/config/GridStatConfig_climo_WMO +++ b/test/config/GridStatConfig_climo_WMO @@ -219,6 +219,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_climo_prob b/test/config/GridStatConfig_climo_prob index b5f4629bf1..a97291af88 100644 --- a/test/config/GridStatConfig_climo_prob +++ b/test/config/GridStatConfig_climo_prob @@ -229,6 +229,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_fourier b/test/config/GridStatConfig_fourier index 6b2ba521f3..ad8c121b23 100644 --- a/test/config/GridStatConfig_fourier +++ b/test/config/GridStatConfig_fourier @@ -185,6 +185,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_grid_weight b/test/config/GridStatConfig_grid_weight index af74b784c7..4cd850faf1 100644 --- a/test/config/GridStatConfig_grid_weight +++ b/test/config/GridStatConfig_grid_weight @@ -170,6 +170,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_interp_shape b/test/config/GridStatConfig_interp_shape index f6a69c5e35..66d1658886 100644 --- a/test/config/GridStatConfig_interp_shape +++ b/test/config/GridStatConfig_interp_shape @@ -152,6 +152,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_mpr_thresh b/test/config/GridStatConfig_mpr_thresh index 4e95022a48..268e24cbc6 100644 --- a/test/config/GridStatConfig_mpr_thresh +++ b/test/config/GridStatConfig_mpr_thresh @@ -217,6 +217,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_no_leap b/test/config/GridStatConfig_no_leap index 3b80dc0330..b33a52c222 100644 --- a/test/config/GridStatConfig_no_leap +++ b/test/config/GridStatConfig_no_leap @@ -159,6 +159,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_prob_as_scalar b/test/config/GridStatConfig_prob_as_scalar index f497641518..8a462ce48a 100644 --- a/test/config/GridStatConfig_prob_as_scalar +++ b/test/config/GridStatConfig_prob_as_scalar @@ -180,6 +180,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_python b/test/config/GridStatConfig_python index 9bb128ac92..3d9059219b 100644 --- a/test/config/GridStatConfig_python +++ b/test/config/GridStatConfig_python @@ -156,6 +156,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_python_mixed b/test/config/GridStatConfig_python_mixed index 38a5f9fa90..9b868a2745 100644 --- a/test/config/GridStatConfig_python_mixed +++ b/test/config/GridStatConfig_python_mixed @@ -164,6 +164,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_rtma b/test/config/GridStatConfig_rtma index b49db75a73..178a1269b1 100644 --- a/test/config/GridStatConfig_rtma +++ b/test/config/GridStatConfig_rtma @@ -160,6 +160,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_rtma_perc_thresh b/test/config/GridStatConfig_rtma_perc_thresh index 55b42b4456..3a739645ad 100644 --- a/test/config/GridStatConfig_rtma_perc_thresh +++ b/test/config/GridStatConfig_rtma_perc_thresh @@ -163,6 +163,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_st4 b/test/config/GridStatConfig_st4 index 00dbf1d8a0..5f210db851 100644 --- a/test/config/GridStatConfig_st4 +++ b/test/config/GridStatConfig_st4 @@ -164,6 +164,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/GridStatConfig_st4_censor b/test/config/GridStatConfig_st4_censor index d0ae97d7d9..1942cfe107 100644 --- a/test/config/GridStatConfig_st4_censor +++ b/test/config/GridStatConfig_st4_censor @@ -173,6 +173,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/ref_config/GridStatConfig_03h b/test/config/ref_config/GridStatConfig_03h index 70f885dd08..dc40220c7b 100644 --- a/test/config/ref_config/GridStatConfig_03h +++ b/test/config/ref_config/GridStatConfig_03h @@ -158,6 +158,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/config/ref_config/GridStatConfig_24h b/test/config/ref_config/GridStatConfig_24h index a66ffcf276..450b7117aa 100644 --- a/test/config/ref_config/GridStatConfig_24h +++ b/test/config/ref_config/GridStatConfig_24h @@ -158,6 +158,7 @@ distance_map = { baddeley_max_dist = NA; fom_alpha = 0.1; zhu_weight = 0.5; + beta_value(n) = n * n / 2.0; } //////////////////////////////////////////////////////////////////////////////// diff --git a/test/hdr/met_10_1.hdr b/test/hdr/met_10_1.hdr index 245eac4203..ba80ba12b8 100644 --- a/test/hdr/met_10_1.hdr +++ b/test/hdr/met_10_1.hdr @@ -10,7 +10,7 @@ NBRCNT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_L NBRCTC : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FY_OY FY_ON FN_OY FN_ON NBRCTS : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL BASER BASER_NCL BASER_NCU BASER_BCL BASER_BCU FMEAN FMEAN_NCL FMEAN_NCU FMEAN_BCL FMEAN_BCU ACC ACC_NCL ACC_NCU ACC_BCL ACC_BCU FBIAS FBIAS_BCL FBIAS_BCU PODY PODY_NCL PODY_NCU PODY_BCL PODY_BCU PODN PODN_NCL PODN_NCU PODN_BCL PODN_BCU POFD POFD_NCL POFD_NCU POFD_BCL POFD_BCU FAR FAR_NCL FAR_NCU FAR_BCL FAR_BCU CSI CSI_NCL CSI_NCU CSI_BCL CSI_BCU GSS GSS_BCL GSS_BCU HK HK_NCL HK_NCU HK_BCL HK_BCU HSS HSS_BCL HSS_BCU ODDS ODDS_NCL ODDS_NCU ODDS_BCL ODDS_BCU LODDS LODDS_NCL LODDS_NCU LODDS_BCL LODDS_BCU ORSS ORSS_NCL ORSS_NCU ORSS_BCL ORSS_BCU EDS EDS_NCL EDS_NCU EDS_BCL EDS_BCU SEDS SEDS_NCL SEDS_NCU SEDS_BCL SEDS_BCU EDI EDI_NCL EDI_NCU EDI_BCL EDI_BCU SEDI SEDI_NCL SEDI_NCU SEDI_BCL SEDI_BCU BAGSS BAGSS_BCL BAGSS_BCU GRAD : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FGBAR OGBAR MGBAR EGBAR S1 S1_OG FGOG_RATIO DX DY -DMAP : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FY OY FBIAS BADDELEY HAUSDORFF MED_FO MED_OF MED_MIN MED_MAX MED_MEAN FOM_FO FOM_OF FOM_MIN FOM_MAX FOM_MEAN ZHU_FO ZHU_OF ZHU_MIN ZHU_MAX ZHU_MEAN +DMAP : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL FY OY FBIAS BADDELEY HAUSDORFF MED_FO MED_OF MED_MIN MED_MAX MED_MEAN FOM_FO FOM_OF FOM_MIN FOM_MAX FOM_MEAN ZHU_FO ZHU_OF ZHU_MIN ZHU_MAX ZHU_MEAN G GBETA BETA_VALUE ORANK : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL INDEX OBS_SID OBS_LAT OBS_LON OBS_LVL OBS_ELV OBS PIT RANK N_ENS_VLD N_ENS _VAR_ PCT : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_THRESH _VAR_ PJC : VERSION MODEL DESC FCST_LEAD FCST_VALID_BEG FCST_VALID_END OBS_LEAD OBS_VALID_BEG OBS_VALID_END FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBTYPE VX_MASK INTERP_MTHD INTERP_PNTS FCST_THRESH OBS_THRESH COV_THRESH ALPHA LINE_TYPE TOTAL N_THRESH _VAR_