diff --git a/.github/dummy_for_action b/.github/dummy_for_action new file mode 100644 index 0000000000..91bdf4d63e --- /dev/null +++ b/.github/dummy_for_action @@ -0,0 +1,2 @@ +#update me to add action comment + diff --git a/.github/jobs/copy_diff_files.py b/.github/jobs/copy_diff_files.py index eed5b6a339..40ddd68fbc 100755 --- a/.github/jobs/copy_diff_files.py +++ b/.github/jobs/copy_diff_files.py @@ -9,18 +9,22 @@ LOG_DIR = '/met/logs' +LINE_BREAK = '# # # # # # # # # # # # # # # # # # # # # # # # # # # # # #' + def get_files_with_diffs(log_file): files_to_copy = set() + error_text_list = [] with open(log_file, 'r') as file_handle: file_content = file_handle.read() missing_section, *test_sections = file_content.split( - '\n# # # # # # # # # # # # # # # # # # # # # # # # # # # # # #\n' + f'\n{LINE_BREAK}\n' ) # parse list of missing files if 'ERROR:' in missing_section: + error_text_list.append(missing_section) for missing_group in missing_section.split('ERROR:')[1:]: dir_str, *rel_paths = missing_group.splitlines() dir_str = dir_str.split()[1] @@ -36,13 +40,14 @@ def get_files_with_diffs(log_file): # parse file paths out of sections that have errors error_sections = [item for item in test_sections if 'ERROR:' in item] + error_text_list.extend(error_sections) for error_section in error_sections: for line in error_section.splitlines(): for item in line.split(): if OUTPUT_DIR in item or TRUTH_DIR in item: files_to_copy.add(item) - return files_to_copy + return files_to_copy, error_text_list def copy_files_to_diff_dir(files_to_copy): @@ -69,12 +74,21 @@ def copy_files_to_diff_dir(files_to_copy): os.makedirs(output_dir) shutil.copyfile(filename, output_path) + def main(): log_file = os.path.join(LOG_DIR, 'comp_dir.log') print(f"Parsing {log_file}") - all_files_to_copy = get_files_with_diffs(log_file) + all_files_to_copy, error_text_list = get_files_with_diffs(log_file) copy_files_to_diff_dir(all_files_to_copy) + # print error summary + if error_text_list: + print("\n\n**************\nERROR SUMMARY:\n**************\n") + + print(f'\n{LINE_BREAK}\n'.join(error_text_list)) + + print("End of script") + if __name__ == "__main__": main() diff --git a/.github/jobs/run_diff_docker.sh b/.github/jobs/run_diff_docker.sh index b4095fb21e..7aa1b4c1f8 100755 --- a/.github/jobs/run_diff_docker.sh +++ b/.github/jobs/run_diff_docker.sh @@ -36,6 +36,8 @@ if [ $? != 0 ]; then fi if [ "$(ls -A ${LOCAL_DIFF_DIR})" ]; then + cat ${LOCAL_LOG_DIR}/copy_diff_files.log + echo "ERROR: Differences exist in the output" # only exit non-zero (job fails) if not updating truth data diff --git a/met/docs/Users_Guide/appendixA.rst b/met/docs/Users_Guide/appendixA.rst index d5e3752ecf..7bf41e810b 100644 --- a/met/docs/Users_Guide/appendixA.rst +++ b/met/docs/Users_Guide/appendixA.rst @@ -1,15 +1,17 @@ .. _appendixA: +******************************** Appendix A FAQs & How do I ... ? -================================ +******************************** Frequently Asked Questions -__________________________ +========================== File-IO -~~~~~~~ +------- -**Q. File-IO - How do I improve the speed of MET tools using Gen-Vx-Mask?** +Q. How do I improve the speed of MET tools using Gen-Vx-Mask? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. The main reason to run gen_vx_mask is to make the MET @@ -23,7 +25,8 @@ recomputing the mask when each MET statistics tool is run. If the polyline only contains a small number of points or the grid is sparse running gen_vx_mask first would only save a second or two. -**Q. File-IO - How do I use map_data?** +Q. How do I use map_data? +^^^^^^^^^^^^^^^^^^^^^^^^^ A. The MET repository includes several map data files. Users can modify which @@ -64,7 +67,8 @@ in the configuration string on the command line: "${MET_BUILD_BASE}/data/map/admin_by_country/admin_China_data"; } \ ]; }' -**Q. FILE-IO - How can I understand the number of matched pairs?** +Q. How can I understand the number of matched pairs? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. Statistics are computed on matched forecast/observation pairs data. @@ -113,7 +117,8 @@ none of the point observations match the variable name requested in the configuration file. So all of the 1166 observations are rejected for the same reason. -**Q. FILE-IO - What types of NetCDF files can MET read?** +Q. What types of NetCDF files can MET read? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. There are three flavors of NetCDF that MET can read directly. @@ -137,7 +142,8 @@ While MET's point2grid tool does support some satellite data inputs, it is limited. Using python embedding is another option for handling new datasets not supported natively by MET. -**Q. FILE-IO - How do I choose a time slice in a NetCDF file?** +Q. How do I choose a time slice in a NetCDF file? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. When processing NetCDF files, the level information needs to be @@ -157,7 +163,8 @@ Let's use plot_data_plane as an example: Assuming that the first array is the time, this will select the 6-th time slice of the APCP data and plot it since these indices are 0-based. -**Q. FILE-IO - How do I use the UNIX time conversion?** +Q. How do I use the UNIX time conversion? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. Regarding the timing information in the NetCDF variable attributes: @@ -195,7 +202,8 @@ subset the TRMM data if needed. Look for the section of it titled "Output domain specification" and define the lat/lon's that needs to be included in the output. -**Q. Does MET use a fixed-width output format for its ASCII output files?** +Q. Does MET use a fixed-width output format for its ASCII output files? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. MET does not use the Fortran-like fixed width format in its @@ -218,7 +226,8 @@ If a fixed-width format is needed, the easiest option would be writing a script to post-process the MET output into the fixed-width format that is needed or that the code expects. -**Q. Do the ASCII output files created by MET use scientific notation?** +Q. Do the ASCII output files created by MET use scientific notation? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. By default, the ASCII output files created by MET make use of @@ -233,12 +242,11 @@ MET to disable the use of scientific notation. That enhancement is planned for a future release. Gen-Vx-Mask -~~~~~~~~~~~ +----------- -**Q. Gen-Vx-Mask - I have a list of stations to use for verification. -I also have a poly region defined. If I specify both of these should -the result be a union of them?** - +Q. I have a list of stations to use for verification. I also have a poly region defined. If I specify both of these should the result be a union of them? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. These settings are defined in the "mask" section of the Point-Stat configuration file. You can define masking regions in one of 3 ways, @@ -266,7 +274,8 @@ If so, your options are: Then aggregate the results together by running a Stat-Analysis job. -**Q. Gen-Vx-Mask - How do I define a masking region with a GFS file?** +Q. How do I define a masking region with a GFS file? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. Grab a sample GFS file: @@ -301,9 +310,10 @@ are obvious problems with the latitude and longitude values used to define that mask for Poland. Grid-Stat -~~~~~~~~~ +--------- -**Q. Grid-Stat - How do I define a complex masking region?** +Q. How do I define a complex masking region? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. A user can define intersections and unions of multiple fields to define masks. @@ -359,8 +369,8 @@ apply complex masking logic and then pass the output mask file to Grid-Stat in its configuration file. -**Q. Grid-Stat - How do I use neighborhood methods to compute fraction -skill score?** +Q. How do I use neighborhood methods to compute fraction skill score? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. A common application of fraction skill score (FSS) is comparing forecast @@ -400,7 +410,8 @@ Setting vld_thresh = 1.0 will ensure that FSS will only be computed at points where all NxN values contain valid data. Setting it to 0.5 only requires half of them. -**Q. Grid-Stat - Is an example of verifying forecast probabilities?** +Q. Is an example of verifying forecast probabilities? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. There is an example of verifying probabilities in the test scripts @@ -443,8 +454,8 @@ values between 0 and 1. Note that if the probability data contains values from 0 to 100, MET automatically divides by 100 to rescale to the 0 to 1 range. -**Q. What is an example of using Grid-Stat with regridding and masking -turned on?** +Q. What is an example of using Grid-Stat with regridding and masking turned on? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. Run Grid-Stat using the following commands and the attached config file @@ -503,9 +514,9 @@ gen_vx_mask tool and pass the NetCDF output of that tool to grid_stat. The advantage to gen_vx_mask is that it will make grid_stat run a bit faster. It can be used to construct much more complex masking areas. -**Q. How do I use one mask for the forecast field and a different -mask for the observation field??** - +Q. How do I use one mask for the forecast field and a different mask for the observation field? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. You can't define different masks for the forecast and observation fields in MET tools. MET only lets you @@ -577,9 +588,10 @@ In the resulting plot, anywhere you see the pink value of 10, that's where gen_vx_mask has masked out the grid point. Pcp-Combine -~~~~~~~~~~~ +----------- -**Q. Pcp-Combine - How do I add and subtract with Pcp-Combine?** +Q. How do I add and subtract with Pcp-Combine? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. An example of running the MET pcp_combine tool to put NAM 3-hourly @@ -616,8 +628,8 @@ same file format, and can use the same configuration file settings for the other MET tools (grid_stat, mode, etc.). If the NAM files are a mix of GRIB and NetCDF, the logic would need to be a bit more complicated. -**Q. Pcp-Combine - How do I combine 12-hour accumulated precipitation -from two different initialization times?** +Q. How do I combine 12-hour accumulated precipitation from two different initialization times? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. The "-sum" command assumes the same initialization time. Use the "-add" @@ -658,7 +670,8 @@ Here are 3 commands you could use to plot these data files: plot_data_plane sum.nc sum.ps 'name="APCP_24"; level="(*,*)";' -**Q. Pcp-Combine - How do I correct a precipitation time range?** +Q. How do I correct a precipitation time range? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. Typically, accumulated precipitation is stored in GRIB files using an @@ -707,8 +720,8 @@ Here is an example: The resulting file should have the accumulation listed at 24h rather than 0-24. -**Pcp-Combine - How do I use Pcp-Combine as a pass-through to simply reformat -from GRIB to NetCDF or to change output variable name?** +Q. How do I use Pcp-Combine as a pass-through to simply reformat from GRIB to NetCDF or to change output variable name? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. The pcp_combine tool is typically used to modify the accumulation interval @@ -745,7 +758,8 @@ appear in the output of downstream tools: 'name="REFC"; level="L0"; GRIB1_ptv=129; lead_time="120000";' \ forecast.nc -name CompositeReflectivity -**Q. Pcp-Combine - How do I use “-pcprx" to run a project faster?** +Q. How do I use “-pcprx" to run a project faster? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. To run a project faster, the “-pcprx” option may be used to narrow the @@ -765,7 +779,8 @@ examples: -pcpdir /scratch4/BMC/shout/ptmp/Andrew.Kren/pre2016c3_corr/temp \ -pcprx 'pgbq[0-9][0-9].gfs.2016022118' -v 3 -**Q. Pcp-Combine - How do I enter the time format correctly?** +Q. How do I enter the time format correctly? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. Here is an **incorrect example** of running pcp_combine with sub-hourly @@ -796,9 +811,9 @@ time strings, like this: pcp_combine -subtract forecast.grb 005500 \ forecast2.grb 000500 forecast.nc -field APCP -**Q. Pcp-Combine - How do I use Pcp-Combine when my GRIB data doesn't have the -appropriate accumulation interval time range indicator?** - +Q. How do I use Pcp-Combine when my GRIB data doesn't have the appropriate accumulation interval time range indicator? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. Run wgrib on the data files and the output is listed below: @@ -855,9 +870,9 @@ Some things to point out here: chosen will be used in the Grid-Stat, Point-Stat, or MODE config file to tell that tool what variable to process. -**Q. Pcp_Combine - How do I use “-sum”, “-add”, and “-subtract“ to achieve -the same accumulation interval?** - +Q. How do I use “-sum”, “-add”, and “-subtract“ to achieve the same accumulation interval? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. Here is an example of using pcp_combine to put GFS into 24- hour intervals for comparison against 24-hourly StageIV precipitation with GFS data @@ -921,14 +936,16 @@ This example explicitly tells pcp_combine which files to read and what accumulation interval (6 hours) to extract from them. The resulting output should be identical to the output of the "-sum" command. -**Q. Pcp-Combine - What is the difference between “-sum” vs. “-add”?** +Q. What is the difference between “-sum” vs. “-add”? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. The -sum and -add options both do the same thing. It's just that '-sum' could find files more quickly with the use of the -pcprx flag. This could also be accomplished by using a calling script. -**Q. Pcp-Combine - How do I select a specific GRIB record?** +Q. How do I select a specific GRIB record? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. In this example, record 735 needs to be selected. @@ -943,9 +960,10 @@ Instead of having the level as "L0", tell it to use "R735" to select grib record 735. Plot-Data-Plane -~~~~~~~~~~~~~~~ +--------------- -**Q. Plot-Data-Plane - How do I inspect Gen-Vx-Mask output?** +Q. How do I inspect Gen-Vx-Mask output? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. Check to see if the call to Gen-Vx-Mask actually did create good output @@ -971,8 +989,9 @@ file is what the user expects. It always a good idea to start with plot_data_plane when working with data to make sure MET is plotting the data correctly and in the expected location. -**Q. Plot-Data-Plane - How do I specify the GRIB version?** - +Q. How do I specify the GRIB version? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. When MET reads Gridded data files, it must determine the type of file it's reading. The first thing it checks is the suffix of the file. @@ -1002,8 +1021,8 @@ MET configuration files (i.e. Grid-Stat, MODE, and so on) that you use: -plot_range 0 100 -**Q. Plot-Data-Plane - How do I test the variable naming convention? -(Record number example)** +Q. How do I test the variable naming convention? (Record number example.) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. Make sure MET can read GRIB2 data. Plot the data from that GRIB2 file @@ -1022,8 +1041,9 @@ contain temperature data and 2-meters. Here's some wgrib2 output: The GRIB id info has been the same between records 1 and 2. -**Q. Plot-Data-Plane - How do I compute and verify wind speed?** - +Q. How do I compute and verify wind speed? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. Here's how to compute and verify wind speed using MET. Good news, MET already includes logic for deriving wind speed on the fly. The GRIB @@ -1053,10 +1073,11 @@ In the second one, this won't appear since wind speed already exists in the RTMA file. Stat-Analysis -~~~~~~~~~~~~~ - -**Q. Stat-Analysis - How does '-aggregate_stat' work?** +------------- +Q. How does '-aggregate_stat' work? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. In Stat-Analysis, there is a "-vx_mask" job filtering option. That option reads the VX_MASK column from the input STAT lines and applies string @@ -1098,10 +1119,9 @@ of the unique values found in that column. Presumably, all the input VX_MASK columns say "FULL" so that's what the output would say. Use "-set_hdr" to explicitly set the output value. -**Q. Stat-Analysis - What is the best way to average the FSS scores -within several days or even several months using -'Aggregate to Average Scores'?** - +Q. What is the best way to average the FSS scores within several days or even several months using 'Aggregate to Average Scores'? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. Below is the best way to aggregate together the Neighborhood Continuous (NBRCNT) lines across multiple days, specifically the fractions skill @@ -1129,8 +1149,9 @@ combination of those header column entries. The output is printed to the screen, or use the "-out_stat" option to also write the aggregated output to a file named "agg_nbrcnt.txt". -**Q. Stat-Analysis - How do I use '-by' to capture unique entries?** - +Q. How do I use '-by' to capture unique entries? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. Here is a stat-analysis job that could be used to run, read the MPR lines, define the probabilistic forecast thresholds, define the single observation @@ -1149,8 +1170,9 @@ to run the job separately for each unique entry found in the FCST_VAR column. The output statistics are written to "out_pstd.txt". -**Q. Stat-Analysis - How do I use '-filter' to refine my output?** - +Q. How do I use '-filter' to refine my output? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. Here is an example of running a Stat-Analysis filter job to discard any CNT lines (continuous statistics) where the forecast rate and observation @@ -1180,8 +1202,9 @@ cases without having to modify the source code. This job reads find 56 CTS lines, but only keeps 36 of them where both the BASER and FMEAN columns are at least 0.05. -**Q. Stat-Analysis - How do I use the “-by” flag to stratify results?** - +Q. How do I use the “-by” flag to stratify results? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. Adding "-by FCST_VAR" is a great way to associate a single value, of say RMSE, with each of the forecast variables (UGRD,VGRD and WIND). @@ -1199,8 +1222,9 @@ Run the following job on the output from Grid-Stat generated when the The resulting cnt.txt file includes separate output for 6 different FCST_VAR values at different levels. -**Q. Stat-Analysis - How do I speed up run times?** - +Q. How do I speed up run times? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. By default, Stat-Analysis has two options enabled which slow it down. Disabling these two options will create quicker run times: @@ -1234,10 +1258,11 @@ Adding the "-by FCST_VAR" option to compute stats for all variables and runs quickly. TC-Stat -~~~~~~~ - -**Q. TC-Stat - How do I use the “-by” flag to stratify results?** +------- +Q. How do I use the “-by” flag to stratify results? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. To perform tropical cyclone evaluations for multiple models use the "-by AMODEL" option with the tc_stat tool. Here is an example. @@ -1257,8 +1282,9 @@ all grouped together. This will result in all 48 hour HWFI and H3WI track forecasts to be aggregated (statistics and scores computed) for each model separately. -**Q. TC-Stat - How do I use rapid intensification verification?** - +Q. How do I use rapid intensification verification? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. To get the most output, run something like this: @@ -1297,10 +1323,11 @@ To stratify your results by lead time, you could add the "-by LEAD" option. -out_line_type CTC,CTS,MPR Utilities -~~~~~~~~~ - -**Q. Utilities - What would be an example of scripting to call MET?** +--------- +Q. What would be an example of scripting to call MET? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. The following is an example of how to call MET from a bash script including passing in variables. This shell script is listed below to run @@ -1322,8 +1349,9 @@ and call convert to reformat from PostScript to PNG. done -**Q. Utility - How do I convert TRMM data files?** - +Q. How do I convert TRMM data files? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. Here is an example of NetCDF that the MET software is not expecting. Here is an option for accessing that same TRMM data, following links from the @@ -1371,8 +1399,9 @@ It may be possible that the domain of the data is smaller. Here are some options That tells Grid-Stat to automatically regrid the TRMM observations to the model domain. -**Q. Other Utilities - How do I convert a PostScript to png?** - +Q. How do I convert a PostScript to png? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. Use the linux “convert” tool to convert a Plot-Data-Plane PostScript file to a png: @@ -1392,8 +1421,9 @@ seperate .png with the following naming convention: mode_out-0.png, mode_out-1.png, mode_out-2.png, etc. -**Q. Utility - How does pairwise differences using plot_tcmpr.R work?** - +Q. How does pairwise differences using plot_tcmpr.R work? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. One necessary step in computing pairwise differences is "event equalizing" the data. This means extracting a subset of cases that are common to @@ -1426,10 +1456,11 @@ pairwise differences that are needed. Miscellaneous -~~~~~~~~~~~~~ - -**Q. Regrid-Data-Plane - How do I define a LatLon grid?** +------------- +Q. Regrid-Data-Plane - How do I define a LatLon grid? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. Here is an example of the NetCDF variable attributes that MET uses to define a LatLon grid: @@ -1457,9 +1488,9 @@ Use ncdump to look at the attributes. As an exercise, try defining these global attributes (and removing the other projection-related ones) and then try again. -**Q. Pre-processing - How do I use wgrib2, pcp_combine regrid and -reformat to format NetCDF files?** - +Q. Pre-processing - How do I use wgrib2, pcp_combine regrid and reformat to format NetCDF files? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. If you are extracting only one or two fields from a file, using MET's Regrid-Data-Plane can be used to generate a Lat-Lon projection. If @@ -1488,9 +1519,9 @@ Then the output NetCDF file does not have this problem: :lat_ll = "25.000000 degrees_north" ; :lon_ll = "112.000000 degrees_east" ; -**Q. TC-Pairs - How do I get rid of WARNING: TrackInfo Using Specify -Model Suffix?** - +Q. TC-Pairs - How do I get rid of WARNING: TrackInfo Using Specify Model Suffix? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. Below is a command example to run: @@ -1574,8 +1605,9 @@ model on the same set of storms. They might be using the same ATCF ID in all their output. But this enables them to distinguish the output in tc_pairs. -**Q. Why is the grid upside down?** - +Q. Why is the grid upside down? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. The user provides a gridded data file to MET and it runs without error, but the data is packed upside down. @@ -1601,7 +1633,7 @@ entry is a dictionary containing information about how to handle input gridded data files. The "regird" entry specifies regridding logic and has a "to_grid" entry that can be set to NONE, FCST, OBS, a named grid, the path to a gridded data file defining the grid, or an explicit grid -specification string. See the :ref:`regrid` entry in +specification string. See the :ref:`regrid` entry in the Configuration File Overview in the MET User's Guide for a more detailed description of the configuration file entries that control automated regridding. @@ -1610,28 +1642,31 @@ A single model level can be plotted using the plot_data_plane utility. This tool can assist the user by showing the data to be verified to ensure that times and locations matchup as expected. -**Q. Why was the MET written largely in C++ instead of FORTRAN?** - +Q. Why was the MET written largely in C++ instead of FORTRAN? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. MET relies upon the object-oriented aspects of C++, particularly in using the MODE tool. Due to time and budget constraints, it also makes use of a pre-existing forecast verification library that was developed at NCAR. -**Q. How does MET differ from the previously mentioned existing -verification packages?** +Q. How does MET differ from the previously mentioned existing verification packages? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. MET is an actively maintained, evolving software package that is being made freely available to the public through controlled version releases. -**Q. Will the MET work on data in native model coordinates?** - +Q. Will the MET work on data in native model coordinates? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. No - it will not. In the future, we may add options to allow additional model grid coordinate systems. -**Q. How do I get help if my questions are not answered in the User's Guide?** +Q. How do I get help if my questions are not answered in the User's Guide? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A. First, look on our @@ -1640,8 +1675,9 @@ If that doesn't answer your question, create a post in the `METplus GitHub Discussions Forum `_. -**Q. What graphical features does MET provide?** - +Q. What graphical features does MET provide? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. MET provides some :ref:`plotting and graphics support`. The plotting tools, including plot_point_obs, plot_data_plane, and plot_mode_field, can @@ -1667,15 +1703,16 @@ you create your own scripts, we encourage you to submit them to us through the `METplus GitHub Discussions Forum `_ so that we can post them for other users. -**Q. How do I find the version of the tool I am using?** - +Q. How do I find the version of the tool I am using? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. Type the name of the tool followed by **-version**. For example, type “pb2nc **-version**”. -**Q. What are MET's conventions for latitude, longitude, azimuth and -bearing angles?** - +Q. What are MET's conventions for latitude, longitude, azimuth and bearing angles? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + A. MET considers north latitude and east longitude positive. Latitudes have range from :math:`-90^\circ` to :math:`+90^\circ`. Longitudes have @@ -1687,7 +1724,7 @@ from the north. .. _Troubleshooting: Troubleshooting -_______________ +=============== The first place to look for help with individual commands is this User's Guide or the usage statements that are provided with the tools. @@ -1697,7 +1734,8 @@ scripts available in the MET's *scripts/* directory show examples of how one might use these commands on example datasets. Here are suggestions on other things to check if you are having problems installing or running MET. -**MET won't compile** +MET won't compile +----------------- * Have you specified the locations of NetCDF, GNU Scientific Library, and BUFRLIB, and optional additional libraries using corresponding @@ -1709,7 +1747,8 @@ on other things to check if you are having problems installing or running MET. * Are you using NetCDF version 3.4 or version 4? Currently, only NetCDF version 3.6 can be used with MET. -**BUFRLIB Errors during MET installation** +BUFRLIB Errors during MET installation +-------------------------------------- .. code-block:: none @@ -1742,7 +1781,8 @@ After doing that, please try recompiling MET. If it fails, please send met_help@ucar.edu the following log files. "make_install.log" as well as "config.log". -**Command line double quotes** +Command line double quotes +-------------------------- Single quotes, double quotes, and escape characters can be difficult for MET to parse. If there are problems, especially in Python code, try @@ -1755,7 +1795,8 @@ breaking the command up like the below example. 'G003', '/h/data/global/WXQC/data/met/nc_mdl/umm/1701150006', '- field', '\'name="HGT"; level="P500";\'', '-v', '6'] -**Environment variable settings** +Environment variable settings +----------------------------- In the below incorrect example for many environment variables have both the main variable set and the INC and LIB variables set: @@ -1786,7 +1827,8 @@ Our online tutorial can help figure out what should be set and what the value should be: https://met.readthedocs.io/en/latest/Users_Guide/installation.html -**NetCDF install issues** +NetCDF install issues +--------------------- This example shows a problem with NetCDF in the make_install.log file: @@ -1811,8 +1853,8 @@ NetCDF library files are in */home/username/local/lib*, unset the MET_NETCDF environment variable, then run "make clean", reconfigure, and then run "make install" and "make test" again. -**Error while loading shared libraries** - +Error while loading shared libraries +------------------------------------ * Add the lib dir to your LD_LIBRARY_PATH. For example, if you receive the following error: "./mode_analysis: error while loading shared @@ -1821,7 +1863,8 @@ and then run "make install" and "make test" again. gsl lib (for example, */home/user/MET/gsl-2.1/lib*) to your LD_LIBRARY_PATH. -**General troubleshooting** +General troubleshooting +----------------------- * For configuration files used, make certain to use empty square brackets (e.g. [ ]) to indicate no stratification is desired. Do NOT use empty @@ -1833,7 +1876,7 @@ and then run "make install" and "make test" again. level to 4 or 5 prints much more diagnostic information to the screen. Where to get help -_________________ +================= If none of the above suggestions have helped solve your problem, help is available through the @@ -1841,7 +1884,7 @@ is available through the How to contribute code -______________________ +====================== If you have code you would like to contribute, we will gladly consider your contribution. Please create a post in the diff --git a/met/docs/Users_Guide/appendixB.rst b/met/docs/Users_Guide/appendixB.rst index 605712ca43..2fb620adf5 100644 --- a/met/docs/Users_Guide/appendixB.rst +++ b/met/docs/Users_Guide/appendixB.rst @@ -1,10 +1,11 @@ .. _appendixB: +************************************************ Appendix B Map Projections, Grids, and Polylines -================================================ +************************************************ Map Projections -_______________ +=============== The following map projections are currently supported in MET: @@ -23,7 +24,7 @@ The following map projections are currently supported in MET: * Gaussian Projection Grid Specification Strings -__________________________ +========================== Several configuration file and command line options support the definition of grids as a grid specification string. A description of the that string for each of the supported grid types is provided below. @@ -82,14 +83,14 @@ For a Gaussian grid, the syntax is The parameters **Nx** and **Ny** are as before, while **lon_zero** defines the first longitude. Grids -_____ +===== The majority of NCEP's pre-defined grids that reside on one of the projections listed above are implemented in MET. The user may specify one of these NCEP grids in the configuration files as "GNNN" where NNN is the 3-digit NCEP grid number. Defining a new masking grid in MET would involve modifying the vx_data_grids library and recompiling. Please see `NCEP's website for a description and plot of these predefined grids `_. Polylines for NCEP Regions -__________________________ +========================== Many of NCEP's pre-defined verification regions are implemented in MET as lat/lon polyline files. The user may specify one of these NCEP verification regions in the configuration files by pointing to the lat/lon polyline file in the installed *share/met/poly* directory. Users may also easily define their own lat/lon polyline files. diff --git a/met/docs/Users_Guide/appendixC.rst b/met/docs/Users_Guide/appendixC.rst index d6523fb1c2..c912032647 100644 --- a/met/docs/Users_Guide/appendixC.rst +++ b/met/docs/Users_Guide/appendixC.rst @@ -1,13 +1,14 @@ .. _appendixC: +******************************** Appendix C Verification Measures -================================ +******************************** This appendix provides specific information about the many verification statistics and measures that are computed by MET. These measures are categorized into measures for categorical (dichotomous) variables; measures for continuous variables; measures for probabilistic forecasts and measures for neighborhood methods. While the continuous, categorical, and probabilistic statistics are computed by both the Point-Stat and Grid-Stat tools, the neighborhood verification measures are only provided by the Grid-Stat tool. Which statistics are the same, but with different names? -________________________________________________________ +======================================================== .. list-table:: Statistics in MET and other names they have been published under. :widths: auto @@ -52,7 +53,7 @@ ________________________________________________________ .. _categorical variables: MET verification measures for categorical (dichotomous) variables -_________________________________________________________________ +================================================================= The verification statistics for dichotomous variables are formulated using a contingency table such as the one shown in :numref:`table_2X2`. In this table f represents the forecasts and o represents the observations; the two possible forecast and observation values are represented by the values 0 and 1. The values in :numref:`table_2X2` are counts of the number of occurrences of the four possible combinations of forecasts and observations. @@ -94,12 +95,12 @@ The values in :numref:`table_2X2` can also be used to compute the F, O, and H re The categorical verification measures produced by the Point-Stat and Grid-Stat tools are described in the following subsections. They are presented in the order shown in :numref:`table_PS_format_info_FHO` through :numref:`table_PS_format_info_CTS_cont`. TOTAL -~~~~~ +----- The total number of forecast-observation pairs, **T**. Base rate -~~~~~~~~~ +--------- Called "O_RATE" in FHO output :numref:`table_PS_format_info_FHO` @@ -108,7 +109,7 @@ Called "BASER" in CTS output :numref:`table_PS_format_info_CTS` The base rate is defined as :math:`\bar{o} = \frac{n_{11} + n_{01}}{T} = \frac{n_{.1}}{T}.` This value is also known as the sample climatology, and is the relative frequency of occurrence of the event (i.e., o = 1). The base rate is equivalent to the "O" value produced by the NCEP Verification System. Mean forecast -~~~~~~~~~~~~~ +------------- Called "F_RATE" in FHO output :numref:`table_PS_format_info_FHO` @@ -119,7 +120,7 @@ The mean forecast value is defined as :math:`\bar{f} = \frac{n_{11} + n_{10}}{T} This statistic is comparable to the base rate and is the relative frequency of occurrence of a forecast of the event (i.e., **f = 1**). The mean forecast is equivalent to the "F" value computed by the NCEP Verification System. Accuracy -~~~~~~~~ +-------- Called "ACC" in CTS output :numref:`table_PS_format_info_CTS` @@ -130,7 +131,7 @@ Accuracy for a 2x2 contingency table is defined as That is, it is the proportion of forecasts that were either hits or correct rejections - the fraction that were correct. Accuracy ranges from 0 to 1; a perfect forecast would have an accuracy value of 1. Accuracy should be used with caution, especially for rare events, because it can be strongly influenced by large values of :math:`\mathbf{n_{00}}`. Frequency Bias -~~~~~~~~~~~~~~ +-------------- Called "FBIAS" in CTS output :numref:`table_PS_format_info_CTS` @@ -141,7 +142,7 @@ Frequency Bias is the ratio of the total number of forecasts of an event to the A "good" value of Frequency Bias is close to 1; a value greater than 1 indicates the event was forecasted too frequently and a value less than 1 indicates the event was not forecasted frequently enough. H_RATE -~~~~~~ +------ Called "H_RATE" in FHO output :numref:`table_PS_format_info_FHO` @@ -158,7 +159,7 @@ H_RATE is defined as H_RATE is equivalent to the H value computed by the NCEP verification system. H_RATE ranges from 0 to 1; a perfect forecast would have H_RATE = 1. Probability of Detection (POD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ Called "PODY" in CTS output :numref:`table_PS_format_info_CTS` @@ -170,7 +171,7 @@ POD is defined as It is the fraction of events that were correctly forecasted to occur. POD is also known as the hit rate. POD ranges from 0 to 1; a perfect forecast would have POD = 1. Probability of False Detection (POFD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------- Called "POFD" in CTS output :numref:`table_PS_format_info_CTS` @@ -182,7 +183,7 @@ POFD is defined as It is the proportion of non-events that were forecast to be events. POFD is also often called the False Alarm Rate. POFD ranges from 0 to 1; a perfect forecast would have POFD = 0. Probability of Detection of the non-event (PODn) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------ Called "PODN" in CTS output :numref:`table_PS_format_info_CTS` @@ -193,7 +194,7 @@ PODn is defined as It is the proportion of non-events that were correctly forecasted to be non-events. Note that PODn = 1 - POFD. PODn ranges from 0 to 1. Like POD, a perfect forecast would have PODn = 1. False Alarm Ratio (FAR) -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- Called "FAR" in CTS output :numref:`table_PS_format_info_CTS` @@ -204,7 +205,7 @@ FAR is defined as It is the proportion of forecasts of the event occurring for which the event did not occur. FAR ranges from 0 to 1; a perfect forecast would have FAR = 0. Critical Success Index (CSI) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------- Called "CSI" in CTS output :numref:`table_PS_format_info_CTS` @@ -215,7 +216,7 @@ CSI is defined as It is the ratio of the number of times the event was correctly forecasted to occur to the number of times it was either forecasted or occurred. CSI ignores the "correct rejections" category (i.e., :math:`\mathbf{n_{00}}`). CSI is also known as the Threat Score (TS). CSI can also be written as a nonlinear combination of POD and FAR, and is strongly related to Frequency Bias and the Base Rate. Gilbert Skill Score (GSS) -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- Called "GSS" in CTS output :numref:`table_PS_format_info_CTS` @@ -230,7 +231,7 @@ where GSS is also known as the Equitable Threat Score (ETS). GSS values range from -1/3 to 1. A no-skill forecast would have GSS = 0; a perfect forecast would have GSS = 1. Hanssen-Kuipers Discriminant (HK) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------- Called "HK" in CTS output :numref:`table_PS_format_info_CTS` @@ -243,7 +244,7 @@ More simply, HK = POD :math:`-` POFD. HK is also known as the True Skill Statistic (TSS) and less commonly (although perhaps more properly) as the Peirce Skill Score. HK measures the ability of the forecast to discriminate between (or correctly classify) events and non-events. HK values range between -1 and 1. A value of 0 indicates no skill; a perfect forecast would have HK = 1. Heidke Skill Score (HSS) -~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------ Called "HSS" in CTS output :numref:`table_PS_format_info_CTS` and "HSS" in MCTS output :numref:`table_PS_format_info_MCTS` @@ -270,7 +271,7 @@ where H is the number of forecasts in the correct category and E is the expected HSS can range from minus infinity to 1. A perfect forecast would have HSS = 1. Heidke Skill Score - Expected Correct (HSS_EC) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------------------- Called "HSS_EC" in MCTS output :numref:`table_PS_format_info_MCTS` @@ -283,7 +284,7 @@ The C_2 value is user-configurable with a default value of T divided by the numb HSS_EC can range from minus infinity to 1. A perfect forecast would have HSS_EC = 1. Odds Ratio (OR) -~~~~~~~~~~~~~~~ +--------------- Called "ODDS" in CTS output :numref:`table_PS_format_info_CTS` @@ -294,14 +295,14 @@ OR measures the ratio of the odds of a forecast of the event being correct to th OR can range from 0 to :math:`\infty`. A perfect forecast would have a value of OR = infinity. OR is often expressed as the log Odds Ratio or as the Odds Ratio Skill Score (:ref:`Stephenson, 2000 `). Logarithm of the Odds Ratio (LODDS) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------------- Called "LODDS" in CTS output :numref:`table_PS_format_info_CTS` LODDS transforms the odds ratio via the logarithm, which tends to normalize the statistic for rare events (:ref:`Stephenson, 2000 `). However, it can take values of :math:`\pm\infty` when any of the contingency table counts is 0. LODDS is defined as :math:`\text{LODDS} = ln(OR)`. Odds Ratio Skill Score (ORSS) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------- Called "ORSS" in CTS output :numref:`table_PS_format_info_CTS` @@ -312,7 +313,7 @@ ORSS is a skill score based on the odds ratio. ORSS is defined as ORSS is sometimes also referred to as Yule's Q. (:ref:`Stephenson, 2000 `). Extreme Dependency Score (EDS) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ Called "EDS" in CTS output :numref:`table_PS_format_info_CTS` @@ -323,7 +324,7 @@ The extreme dependency score measures the association between forecast and obser EDS can range from -1 to 1, with 0 representing no skill. A perfect forecast would have a value of EDS = 1. EDS is independent of bias, so should be presented along with the frequency bias statistic (:ref:`Stephenson et al., 2008 `). Extreme Dependency Index (EDI) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ Called "EDI" in CTS output :numref:`table_PS_format_info_CTS` @@ -336,7 +337,7 @@ where *H* and *F* are the Hit Rate and False Alarm Rate, respectively. EDI can range from :math:`-\infty` to 1, with 0 representing no skill. A perfect forecast would have a value of EDI = 1 (:ref:`Ferro and Stephenson, 2011 `). Symmetric Extreme Dependency Score (SEDS) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------------------- Called "SEDS" in CTS output :numref:`table_PS_format_info_CTS` @@ -347,7 +348,7 @@ The symmetric extreme dependency score measures the association between forecast SEDS can range from :math:`-\infty` to 1, with 0 representing no skill. A perfect forecast would have a value of SEDS = 1 (:ref:`Ferro and Stephenson, 2011 `). Symmetric Extremal Dependency Index (SEDI) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------ Called "SEDI" in CTS output :numref:`table_PS_format_info_CTS` @@ -360,14 +361,14 @@ where :math:`H = \frac{n_{11}}{n_{11} + n_{01}}` and :math:`F = \frac{n_{10}}{n_ SEDI can range from :math:`-\infty` to 1, with 0 representing no skill. A perfect forecast would have a value of SEDI = 1. SEDI approaches 1 only as the forecast approaches perfection (:ref:`Ferro and Stephenson, 2011 `). Bias Adjusted Gilbert Skill Score (GSS) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------------- Called "BAGSS" in CTS output :numref:`table_PS_format_info_CTS` BAGSS is based on the GSS, but is corrected as much as possible for forecast bias (:ref:`Brill and Mesinger, 2009 `). Economic Cost Loss Relative Value (ECLV) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------------- Included in ECLV output :numref:`table_PS_format_info_ECLV` @@ -382,14 +383,14 @@ For cost / loss ratio above the base rate, the ECLV is defined as: .. math:: \text{ECLV } = \frac{(cl \ast (h + f)) + m - b}{b \ast (cl - 1)}. MET verification measures for continuous variables -__________________________________________________ +================================================== For continuous variables, many verification measures are based on the forecast error (i.e., **f - o**). However, it also is of interest to investigate characteristics of the forecasts, and the observations, as well as their relationship. These concepts are consistent with the general framework for verification outlined by :ref:`Murphy and Winkler, (1987) `. The statistics produced by MET for continuous forecasts represent this philosophy of verification, which focuses on a variety of aspects of performance rather than a single measure. The verification measures currently evaluated by the Point-Stat tool are defined and described in the subsections below. In these definitions, **f** represents the forecasts, **o** represents the observation, and **n** is the number of forecast-observation pairs. Mean forecast -~~~~~~~~~~~~~ +------------- Called "FBAR" in CNT output :numref:`table_PS_format_info_CNT` @@ -398,7 +399,7 @@ Called "FBAR" in SL1L2 output :numref:`table_PS_format_info_SL1L2` The sample mean forecast, FBAR, is defined as :math:`\bar{f} = \frac{1}{n} \sum_{i=1}^{n} f_i`. Mean observation -~~~~~~~~~~~~~~~~ +---------------- Called "OBAR" in CNT output :numref:`table_PS_format_info_CNT` @@ -407,7 +408,7 @@ Called "OBAR" in SL1L2 output :numref:`table_PS_format_info_SL1L2` The sample mean observation is defined as :math:`\bar{o} = \frac{1}{n} \sum_{i=1}^{n} o_i`. Forecast standard deviation -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- Called "FSTDEV" in CNT output :numref:`table_PS_format_info_CNT` @@ -418,7 +419,7 @@ The sample variance of the forecasts is defined as The forecast standard deviation is defined as :math:`s_f = \sqrt{s_f^2}`. Observation standard deviation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ Called "OSTDEV" in CNT output :numref:`table_PS_format_info_CNT` @@ -429,7 +430,7 @@ The sample variance of the observations is defined as The observed standard deviation is defined as :math:`s_o = \sqrt{s_o^2}`. Pearson Correlation Coefficient -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- Called "PR_CORR" in CNT output :numref:`table_PS_format_info_CNT` @@ -440,7 +441,7 @@ The Pearson correlation coefficient, **r**, measures the strength of linear asso **r** can range between -1 and 1; a value of 1 indicates perfect correlation and a value of -1 indicates perfect negative correlation. A value of 0 indicates that the forecasts and observations are not correlated. Spearman rank correlation coefficient :math:`(\rho_{s})` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------------------------- Called "SP_CORR" in CNT :numref:`table_PS_format_info_CNT` @@ -453,7 +454,7 @@ A simpler formulation of the Spearman-rank correlation is based on differences b Like **r**, the Spearman rank correlation coefficient ranges between -1 and 1; a value of 1 indicates perfect correlation and a value of -1 indicates perfect negative correlation. A value of 0 indicates that the forecasts and observations are not correlated. Kendall's Tau statistic ( :math:`\tau`) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------------- Called "KT_CORR" in CNT output :numref:`table_PS_format_info_CNT` @@ -466,7 +467,7 @@ where :math:`N_C` is the number of "concordant" pairs and :math:`N_D` is the num Like **r** and :math:`\rho_{s}`, Kendall's Tau ( :math:`\tau`) ranges between -1 and 1; a value of 1 indicates perfect association (concordance) and a value of -1 indicates perfect negative association. A value of 0 indicates that the forecasts and observations are not associated. Mean Error (ME) -~~~~~~~~~~~~~~~ +--------------- Called "ME" in CNT output :numref:`table_PS_format_info_CNT` @@ -477,7 +478,7 @@ The Mean Error, ME, is a measure of overall bias for continuous variables; in pa A perfect forecast has ME = 0. Mean Error Squared (ME2) -~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------ Called "ME2" in CNT output :numref:`table_PS_format_info_CNT` @@ -486,21 +487,21 @@ The Mean Error Squared, ME2, is provided to give a complete breakdown of MSE in A perfect forecast has ME2 = 0. Multiplicative Bias -~~~~~~~~~~~~~~~~~~~ +------------------- Called "MBIAS" in CNT output :numref:`table_PS_format_info_CNT` Multiplicative bias is simply the ratio of the means of the forecasts and the observations: :math:`\text{MBIAS} = \bar{f} / \bar{o}` Mean-squared error (MSE) -~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------ Called "MSE" in CNT output :numref:`table_PS_format_info_CNT` MSE measures the average squared error of the forecasts. Specifically, :math:`\text{MSE} = \frac{1}{n}\sum (f_{i} - o_{i})^{2}`. Root-mean-squared error (RMSE) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ Called "RMSE" in CNT output :numref:`table_PS_format_info_CNT` @@ -508,7 +509,7 @@ RMSE is simply the square root of the MSE, :math:`\text{RMSE} = \sqrt{\text{MSE} Scatter Index (SI) -~~~~~~~~~~~~~~~~~~ +------------------ Called "SI" in CNT output :numref:`table_PS_format_info_CNT` @@ -517,12 +518,12 @@ SI is the ratio of the root mean squared error to the average observation value, Smaller values of SI indicate better agreement between the model and observations (less scatter on scatter plot). Standard deviation of the error -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- Called "ESTDEV" in CNT output :numref:`table_PS_format_info_CNT` Bias-Corrected MSE -~~~~~~~~~~~~~~~~~~ +------------------ Called "BCMSE" in CNT output :numref:`table_PS_format_info_CNT` @@ -535,7 +536,7 @@ The standard deviation of the error, :math:`s_{f-o}`, is :math:`s_{f-o} = \sqrt{ Note that the square of the standard deviation of the error (ESTDEV2) is sometimes called the "Bias-corrected MSE" (BCMSE) because it removes the effect of overall bias from the forecast-observation squared differences. Mean Absolute Error (MAE) -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- Called "MAE" in CNT output :numref:`table_PS_format_info_CNT` @@ -544,7 +545,7 @@ The Mean Absolute Error (MAE) is defined as :math:`\text{MAE} = \frac{1}{n} \sum MAE is less influenced by large errors and also does not depend on the mean error. A perfect forecast would have MAE = 0. InterQuartile Range of the Errors (IQR) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------------- Called "IQR" in CNT output :numref:`table_PS_format_info_CNT` @@ -553,7 +554,7 @@ The InterQuartile Range of the Errors (IQR) is the difference between the 75th a IQR is another estimate of spread, similar to standard error, but is less influenced by large errors and also does not depend on the mean error. A perfect forecast would have IQR = 0. Median Absolute Deviation (MAD) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- Called "MAD" in CNT output :numref:`table_PS_format_info_CNT` @@ -562,7 +563,7 @@ The Median Absolute Deviation (MAD) is defined as :math:`\text{MAD} = \text{medi MAD is an estimate of spread, similar to standard error, but is less influenced by large errors and also does not depend on the mean error. A perfect forecast would have MAD = 0. Mean Squared Error Skill Score -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ Called "MSESS" in CNT output :numref:`table_PS_format_info_CNT` @@ -571,27 +572,28 @@ The Mean Squared Error Skill Score is one minus the ratio of the forecast MSE to .. math:: \text{MSESS} = 1 - \frac{\text{MSE}_f}{\text{MSE}_r} Root-mean-squared Forecast Anomaly -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------- Called "RMSFA" in CNT output :numref:`table_PS_format_info_CNT` RMSFA is the square root of the average squared forecast anomaly. Specifically, :math:`\text{RMSFA} = \sqrt{\frac{1}{n} \sum(f_{i} - c_{i})^2}`. Root-mean-squared Observation Anomaly -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------- Called "RMSOA" in CNT output :numref:`table_PS_format_info_CNT` RMSOA is the square root of the average squared observation anomaly. Specifically, :math:`\text{RMSOA} = \sqrt{\frac{1}{n} \sum(o_{i} - c_{i})^2}`. Percentiles of the errors -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- + Called "E10", "E25", "E50", "E75", "E90" in CNT output :numref:`table_PS_format_info_CNT` Percentiles of the errors provide more information about the distribution of errors than can be obtained from the mean and standard deviations of the errors. Percentiles are computed by ordering the errors from smallest to largest and computing the rank location of each percentile in the ordering, and matching the rank to the actual value. Percentiles can also be used to create box plots of the errors. In MET, the 0.10th, 0.25th, 0.50th, 0.75th, and 0.90th quantile values of the errors are computed. Anomaly Correlation Coefficient -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- Called "ANOM_CORR" and "ANOM_CORR_UNCNTR" for centered and uncentered versions in CNT output :numref:`table_PS_format_info_CNT` @@ -616,7 +618,7 @@ The uncentered anomaly correlation coefficient (ANOM_CORR_UNCNTR) which does not Anomaly correlation can range between -1 and 1; a value of 1 indicates perfect correlation and a value of -1 indicates perfect negative correlation. A value of 0 indicates that the forecast and observed anomalies are not correlated. Partial Sums lines (SL1L2, SAL1L2, VL1L2, VAL1L2) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------- :numref:`table_PS_format_info_SL1L2`, :numref:`table_PS_format_info_SAL1L2`, :numref:`table_PS_format_info_VL1L2`, and :numref:`table_PS_format_info_VAL1L2` @@ -627,7 +629,7 @@ The partial sums can be accumulated over individual cases to produce statistics *Minimally sufficient* statistics are those that condense the data most, with no loss of information. Statistics based on L1 and L2 norms allow for good compression of information. Statistics based on other norms, such as order statistics, do not result in good compression of information. For this reason, statistics such as RMSE are often preferred to statistics such as the median absolute deviation. The partial sums are not sufficient for order statistics, such as the median or quartiles. Scalar L1 and L2 values -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- Called "FBAR", "OBAR", "FOBAR", "FFBAR", and "OOBAR" in SL1L2 output :numref:`table_PS_format_info_SL1L2` @@ -647,7 +649,7 @@ These statistics are simply the 1st and 2nd moments of the forecasts, observatio Some of the other statistics for continuous forecasts (e.g., RMSE) can be derived from these moments. Scalar anomaly L1 and L2 values -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- Called "FABAR", "OABAR", "FOABAR", "FFABAR", "OOABAR" in SAL1L2 output :numref:`table_PS_format_info_SAL1L2` @@ -665,7 +667,7 @@ Computation of these statistics requires a climatological value, c. These statis \text{OOABAR} = \text{Mean}[(o - c)^2] = \bar{(o - c)}^2 = \frac{1}{n} \sum_{i=1}^n (o_i - c)^2 Vector L1 and L2 values -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- Called "UFBAR", "VFBAR", "UOBAR", "VOBAR", "UVFOBAR", "UVFFBAR", "UVOOBAR" in VL1L2 output :numref:`table_PS_format_info_VL1L2` @@ -687,7 +689,7 @@ These statistics are the moments for wind vector values, where **u** is the E-W \text{UVOOBAR} = \text{Mean}(u_o^2 + v_o^2) = \frac{1}{n} \sum_{i=1}^n (u_{oi}^2 + v_{oi}^2) Vector anomaly L1 and L2 values -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- Called "UFABAR", "VFABAR", "UOABAR", "VOABAR", "UVFOABAR", "UVFFABAR", "UVOOABAR" in VAL1L2 output :numref:`table_PS_format_info_VAL1L2` @@ -710,7 +712,7 @@ These statistics require climatological values for the wind vector components, : \text{UVOOABAR} = \text{Mean}[(u_o - u_c)^2 + (v_o - v_c)^2] = \frac{1}{n} \sum_{i=1}^n ((u_{oi} - u_c)^2 + (v_{oi} - v_c)^2) Gradient values -~~~~~~~~~~~~~~~ +--------------- Called "TOTAL", "FGBAR", "OGBAR", "MGBAR", "EGBAR", "S1", "S1_OG", and "FGOG_RATIO" in GRAD output :numref:`table_GS_format_info_GRAD` @@ -752,7 +754,7 @@ where the weights are applied at each grid location, with values assigned accord MET verification measures for probabilistic forecasts -_____________________________________________________ +===================================================== The results of the probabilistic verification methods that are included in the Point-Stat, Grid-Stat, and Stat-Analysis tools are summarized using a variety of measures. MET treats probabilistic forecasts as categorical, divided into bins by user-defined thresholds between zero and one. For the categorical measures, if a forecast probability is specified in a formula, the midpoint value of the bin is used. These measures include the Brier Score (BS) with confidence bounds (:ref:`Bradley, 2008 `); the joint distribution, calibration-refinement, likelihood-base rate (:ref:`Wilks, 2011 `); and receiver operating characteristic information. Using these statistics, reliability and discrimination diagrams can be produced. @@ -795,7 +797,7 @@ The verification statistics for probabilistic forecasts of dichotomous variables Reliability -~~~~~~~~~~~ +----------- Called "RELIABILITY" in PSTD output :numref:`table_PS_format_info_PSTD` @@ -804,7 +806,8 @@ A component of the Brier score. Reliability measures the average difference betw .. math:: \text{Reliability} = \frac{1}{T} \sum n_i (p_i - \bar{o}_i)^2 Resolution -~~~~~~~~~~ +---------- + Called "RESOLUTION" in PSTD output :numref:`table_PS_format_info_PSTD` A component of the Brier score that measures how well forecasts divide events into subsets with different outcomes. Larger values of resolution are best since it is desirable for event frequencies in the subsets to be different than the overall event frequency. @@ -812,7 +815,7 @@ A component of the Brier score that measures how well forecasts divide events in .. math:: \text{Resolution} = \frac{1}{T} n_{i.}(\bar{o}_i - \bar{o})^2 Uncertainty -~~~~~~~~~~~ +----------- Called "UNCERTAINTY" in PSTD output :numref:`table_PS_format_info_PSTD` @@ -821,7 +824,7 @@ A component of the Brier score. For probabilistic forecasts, uncertainty is a fu .. math:: \text{Uncertainty} = \frac{n_{.1}}{T}(1 - \frac{n_{.1}}{T}) Brier score -~~~~~~~~~~~ +----------- Called "BRIER" in PSTD output :numref:`table_PS_format_info_PSTD` @@ -838,7 +841,7 @@ BS can be partitioned into three terms: (1) reliability, (2) resolution, and (3) This score is sensitive to the base rate or climatological frequency of the event. Forecasts of rare events can have a good BS without having any actual skill. Since Brier score is a measure of error, smaller values are better. Brier Skill Score (BSS) -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- Called "BSS" and "BSS_SMPL" in PSTD output :numref:`table_PS_format_info_PSTD` @@ -849,7 +852,7 @@ BSS is a skill score based on the Brier Scores of the forecast and a reference f BSS is computed using the climatology specified in the configuration file while BSS_SMPL is computed using the sample climatology of the current set of observations. OY_TP - Observed Yes Total Proportion -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------- Called "OY_TP" in PJC output :numref:`table_PS_format_info_PJC` @@ -858,7 +861,7 @@ This is the cell probability for row **i**, column **j=1** (observed event), a p .. math:: \text{OYTP}(i) = \frac{n_{i1}}{T} = \text{probability}(o_{i1}) ON_TP - Observed No Total Proportion -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------ Called "ON_TP" in PJC output :numref:`table_PS_format_info_PJC` @@ -867,7 +870,7 @@ This is the cell probability for row **i**, column **j=0** (observed non-event), .. math:: \text{ONTP}(i) = \frac{n_{i0}}{T} = \text{probability}(o_{i0}) Calibration -~~~~~~~~~~~ +----------- Called "CALIBRATION" in PJC output :numref:`table_PS_format_info_PJC` @@ -876,7 +879,7 @@ Calibration is the conditional probability of an event given each probability fo .. math:: \text{Calibration}(i) = \frac{n_{i1}}{n_{1.}} = \text{probability}(o_1|p_i) Refinement -~~~~~~~~~~ +---------- Called "REFINEMENT" in PJC output :numref:`table_PS_format_info_PJC` @@ -885,7 +888,7 @@ The relative frequency associated with each forecast probability, sometimes call .. math:: \text{Refinement}(i) = \frac{n_{i.}}{T} = \text{probability}(p_i) Likelihood -~~~~~~~~~~ +---------- Called "LIKELIHOOD" in PJC output :numref:`table_PS_format_info_PJC` @@ -896,7 +899,7 @@ Likelihood is the conditional probability for each forecast category (row) given Likelihood values are also used to create "discrimination" plots that compare the distribution of forecast values for events to the distribution of forecast values for non-events. These plots show how well the forecasts categorize events and non-events. The distribution of forecast values for non-events can be derived from the POFD values computed by MET for the user-specified thresholds. Base Rate -~~~~~~~~~ +--------- Called "BASER" in PJC output :numref:`table_PS_format_info_PJC` @@ -905,7 +908,7 @@ This is the probability of an event for each forecast category :math:`p_i` (row) .. math:: \text{Base Rate}(i) = \frac{n_{i1}}{n_{i.}} = \text{probability}(o_{i1}) Reliability diagram -~~~~~~~~~~~~~~~~~~~ +------------------- The reliability diagram is a plot of the observed frequency of events versus the forecast probability of those events, with the range of forecast probabilities divided into categories. @@ -918,7 +921,7 @@ The ideal forecast (i.e., one with perfect reliability) has conditional observed Example of Reliability Diagram Receiver operating characteristic -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------- MET produces hit rate (POD) and false alarm rate (POFD) values for each user-specified threshold. This information can be used to create a scatter plot of POFD vs. POD. When the points are connected, the plot is generally referred to as the receiver operating characteristic (ROC) curve (also called the "relative operating characteristic" curve). See the area under the ROC curve (AUC) entry for related information. @@ -933,7 +936,7 @@ A ROC curve shows how well the forecast discriminates between two outcomes, so i Example of ROC Curve Area Under the ROC curve (AUC) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ Called "ROC_AUC" in PSTD output :numref:`table_PS_format_info_PSTD` @@ -946,10 +949,10 @@ The area under the curve can be estimated in a variety of ways. In MET, the simp .. _App_C-ensemble: MET verification measures for ensemble forecasts -________________________________________________ +================================================ RPS -~~~ +--- Called "RPS" in RPS output :numref:`table_ES_header_info_es_out_ECNT` @@ -967,7 +970,7 @@ To clarify, :math:`F_1 = f_1` is the first component of :math:`F_m`, :math:`F_2 where :math:`BS_m` is the Brier score for the m-th category (:ref:`Tödter and Ahrens, 2012`). Subsequently, the RPS lends itself to a decomposition into reliability, resolution and uncertainty components, noting that each component is aggregated over the different categories; these are written to the columns named "RPS_REL", "RPS_RES" and "RPS_UNC" in RPS output :numref:`table_ES_header_info_es_out_ECNT`. CRPS -~~~~ +---- Called "CRPS", "CRPSCL", "CRPS_EMP", and "CRPSCL_EMP" in ECNT output :numref:`table_ES_header_info_es_out_ECNT` @@ -986,7 +989,7 @@ The overall CRPS is calculated as the average of the individual measures. In equ The score can be interpreted as a continuous version of the mean absolute error (MAE). Thus, the score is negatively oriented, so smaller is better. Further, similar to MAE, bias will inflate the CRPS. Thus, bias should also be calculated and considered when judging forecast quality using CRPS. CRPS Skill Score -~~~~~~~~~~~~~~~~ +---------------- Called "CRPSS" and "CRPSS_EMP" in ECNT output :numref:`table_ES_header_info_es_out_ECNT` @@ -997,7 +1000,7 @@ The continuous ranked probability skill score (CRPSS) is similar to the MSESS an For the normal distribution fit (CRPSS), the reference CRPS is computed using the climatological mean and standard deviation. For the empirical distribution (CRPSS_EMP), the reference CRPS is computed by sampling from the assumed normal climatological distribution defined by the mean and standard deviation. IGN -~~~ +--- Called "IGN" in ECNT output :numref:`table_ES_header_info_es_out_ECNT` @@ -1008,14 +1011,14 @@ The ignorance score (IGN) is the negative logarithm of a predictive probability Accumulation of the ignorance score for many forecasts is via the average of individual ignorance scores. This average ignorance score is the value output by the MET software. Like many error statistics, the IGN is negatively oriented, so smaller numbers indicate better forecasts. PIT -~~~ +--- Called "PIT" in ORANK output :numref:`table_ES_header_info_es_out_ORANK` The probability integral transform (PIT) is the analog of the rank histogram for a probability distribution forecast (:ref:`Dawid, 1984 `). Its interpretation is the same as that of the verification rank histogram: Calibrated probabilistic forecasts yield PIT histograms that are flat, or uniform. Under-dispersed (not enough spread in the ensemble) forecasts have U-shaped PIT histograms while over-dispersed forecasts have bell-shaped histograms. In MET, the PIT calculation uses a normal distribution fit to the ensemble forecasts. In many cases, use of other distributions would be better. RANK -~~~~ +---- Called "RANK" in ORANK output :numref:`table_ES_header_info_es_out_ORANK` @@ -1024,7 +1027,7 @@ The rank of an observation, compared to all members of an ensemble forecast, is The rank histogram does not provide information about the accuracy of ensemble forecasts. Further, examination of "rank" only makes sense for ensembles of a fixed size. Thus, if ensemble members are occasionally unavailable, the rank histogram should not be used. The PIT may be used instead. SPREAD -~~~~~~ +------ Called "SPREAD" in ECNT output :numref:`table_ES_header_info_es_out_ECNT` @@ -1035,7 +1038,7 @@ The ensemble spread for a single observation is the standard deviation of the en Note that prior to met-9.0.1, the ensemble spread of a spatial masking region was computed as the average of the spread values within that region. This algorithm was corrected in met-9.0.1 to average the ensemble variance values prior to computing the square root. MET verification measures for neighborhood methods -__________________________________________________ +================================================== The results of the neighborhood verification approaches that are included in the Grid-Stat tool are summarized using a variety of measures. These measures include the Fractions Skill Score (FSS) and the Fractions Brier Score (FBS). MET also computes traditional contingency table statistics for each combination of threshold and neighborhood window size. @@ -1072,14 +1075,14 @@ All of these measures are defined in :numref:`categorical variables`. In addition to these standard statistics, the neighborhood analysis provides additional continuous measures, the Fractions Brier Score and the Fractions Skill Score. For reference, the Asymptotic Fractions Skill Score and Uniform Fractions Skill Score are also calculated. These measures are defined here, but are explained in much greater detail in :ref:`Ebert (2008) ` and :ref:`Roberts and Lean (2008) `. :ref:`Roberts and Lean (2008) ` also present an application of the methodology. Fractions Brier Score -~~~~~~~~~~~~~~~~~~~~~ +--------------------- Called "FBS" in NBRCNT output :numref:`table_GS_format_info_NBRCNT` The Fractions Brier Score (FBS) is defined as :math:`\text{FBS} = \frac{1}{N} \sum_N [\langle P_f\rangle_s - \langle P_o\rangle_s]^2`, where N is the number of neighborhoods; :math:`\langle P_{f} \rangle_{s}` is the proportion of grid boxes within a forecast neighborhood where the prescribed threshold was exceeded (i.e., the proportion of grid boxes that have forecast events); and :math:`\langle P_{o}\rangle_{s}` is the proportion of grid boxes within an observed neighborhood where the prescribed threshold was exceeded (i.e., the proportion of grid boxes that have observed events). Fractions Skill Score -~~~~~~~~~~~~~~~~~~~~~ +--------------------- Called "FSS" in NBRCNT output :numref:`table_GS_format_info_NBRCNT` @@ -1090,28 +1093,28 @@ The Fractions Skill Score (FSS) is defined as where the denominator represents the worst possible forecast (i.e., with no overlap between forecast and observed events). FSS ranges between 0 and 1, with 0 representing no overlap and 1 representing complete overlap between forecast and observed events, respectively. Asymptotic Fractions Skill Score -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------- Called "AFSS" in NBRCNT output :numref:`table_GS_format_info_NBRCNT` The Asymptotic Fractions Skill Score (AFSS) is a special case of the Fractions Skill score where the entire domain is used as the single neighborhood. This provides the user with information about the overall frequency bias of forecasts versus observations. The formula is the same as for FSS above, but with N=1 and the neighborhood size equal to the domain. Uniform Fractions Skill Score -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------- Called "UFSS" in NBRCNT output :numref:`table_GS_format_info_NBRCNT` The Uniform Fractions Skill Score (UFSS) is a reference statistic for the Fractions Skill score based on a uniform distribution of the total observed events across the grid. UFSS represents the FSS that would be obtained at the grid scale from a forecast with a fraction/probability equal to the total observed event proportion at every point. The formula is :math:`UFSS = (1 + f_o)/2` (i.e., halfway between perfect skill and random forecast skill) where :math:`f_o` is the total observed event proportion (i.e. observation rate). Forecast Rate -~~~~~~~~~~~~~ +------------- Called "F_rate" in NBRCNT output :numref:`table_GS_format_info_NBRCNT` The overall proportion of grid points with forecast events to total grid points in the domain. The forecast rate will match the observation rate in unbiased forecasts. Observation Rate -~~~~~~~~~~~~~~~~ +---------------- Called "O_rate" in NBRCNT output :numref:`table_GS_format_info_NBRCNT` @@ -1120,7 +1123,7 @@ The overall proportion of grid points with observed events to total grid points .. _App_C-distance_maps: MET verification measures for distance map methods -__________________________________________________ +================================================== The distance map statistics include Baddeley's :math:`\Delta` Metric, a statistic which is a true mathematical metric. The definition of a mathematical metric is included below. @@ -1139,7 +1142,7 @@ It has been argued in :ref:`Gilleland (2017) ` that the second p The results of the distance map verification approaches that are included in the Grid-Stat tool are summarized using a variety of measures. These measures include Baddeley's :math:`\Delta` Metric, the Hausdorff Distance, the Mean-error Distance, Pratt's Figure of Merit, and Zhu's Measure. Their equations are listed below. Baddeley's :math:`\Delta` Metric and Hausdorff Distance -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------------- Called "BADDELEY" and "HAUSDORFF" in the DMAP output :numref:`table_GS_format_info_DMAP` @@ -1155,7 +1158,7 @@ In terms of distance maps, Baddeley's :math:`\Delta` is the :math:`L_{p}` norm o The range for BADDELEY and HAUSDORFF is 0 to infinity, with a score of 0 indicating a perfect forecast. Mean-error Distance -~~~~~~~~~~~~~~~~~~~ +------------------- Called "MED_FO", "MED_OF", "MED_MIN", "MED_MAX", and "MED_MEAN" in the DMAP output :numref:`table_GS_format_info_DMAP` @@ -1178,7 +1181,7 @@ From the distance map perspective, MED *(A,B)* is the average of the values in : The range for MED is 0 to infinity, with a score of 0 indicating a perfect forecast. Pratt's Figure of Merit -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- Called "FOM_FO", "FOM_OF", "FOM_MIN", "FOM_MAX", and "FOM_MEAN" in the DMAP output :numref:`table_GS_format_info_DMAP` @@ -1193,7 +1196,7 @@ Note that :math:`d(s,A)` in the denominator is summed only over the grid squares The range for FOM is 0 to 1, with a score of 1 indicating a perfect forecast. Zhu's Measure -~~~~~~~~~~~~~ +------------- Called "ZHU_FO", "ZHU_OF", "ZHU_MIN", "ZHU_MAX", and "ZHU_MEAN" in the DMAP output :numref:`table_GS_format_info_DMAP` @@ -1208,7 +1211,7 @@ The range for ZHU is 0 to infinity, with a score of 0 indicating a perfect forec .. _App_C-gbeta: :math:`G` and :math:`G_\beta` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------- Called "G" and "GBETA" in the DMAP output :numref:`table_GS_format_info_DMAP` @@ -1229,7 +1232,7 @@ where :math:`\beta > 0` is a user-chosen parameter with a default value of :math The range for :math:`G_\beta` is 0 to 1, with a score of 1 indicating a perfect forecast. Calculating Percentiles -_______________________ +======================= Several of the MET tools make use of percentiles in one way or another. Percentiles can be used as part of the internal computations of a tool, or can be written out as elements of some of the standard verification statistics. There are several widely-used conventions for calculating percentiles however, so in this section we describe how percentiles are calculated in MET. diff --git a/met/docs/Users_Guide/appendixD.rst b/met/docs/Users_Guide/appendixD.rst index 66c578e91a..0c53652e59 100644 --- a/met/docs/Users_Guide/appendixD.rst +++ b/met/docs/Users_Guide/appendixD.rst @@ -2,8 +2,9 @@ .. _App_D-Confidence-Intervals: +******************************* Appendix D Confidence Intervals -=============================== +******************************* A single verification statistic is statistically meaningless without associated uncertainty information in accompaniment. There can be numerous sources of uncertainty associated with such a statistic including observational, physical uncertainties about the underlying processes governing the equation, sample uncertainty, etc. Although all of the sources of uncertainty can be important, the most heavily researched, and easiest to calculate, is that of sampling uncertainty. It is this source of uncertainty that can presently be obtained with MET, and the techniques for deriving these estimates are described here. Sampling uncertainty through MET is gleaned by way of confidence intervals (CIs) as these are generally most informative. A :math:`(1 - \alpha) \cdot 100\%` confidence interval is interpreted, somewhat awkwardly, in the following way. If the test were repeated 100 times (so that we have 100 such intervals), then we expect the true value of the statistics to fall inside :math:`(1-\alpha)\cdot 100` of these intervals. For example, if :math:`\alpha=0.05` then we expect the true value to fall within 95 of the intervals. diff --git a/met/docs/Users_Guide/appendixE.rst b/met/docs/Users_Guide/appendixE.rst index a50afe1703..4ec8ff2a9a 100644 --- a/met/docs/Users_Guide/appendixE.rst +++ b/met/docs/Users_Guide/appendixE.rst @@ -1,7 +1,8 @@ .. _appendixE: +********************** Appendix E WWMCA Tools -====================== +********************** There are two WWMCA tools available. The WWMCA-Plot tool makes a PostScript plot of one or more WWMCA cloud percent files and the WWMCA-Regrid tool regrids WWMCA cloud percent files and reformats them into netCDF files that the other MET tools can read. diff --git a/met/docs/Users_Guide/appendixF.rst b/met/docs/Users_Guide/appendixF.rst index 586c264c9e..2649fb4072 100644 --- a/met/docs/Users_Guide/appendixF.rst +++ b/met/docs/Users_Guide/appendixF.rst @@ -1,15 +1,16 @@ .. _appendixF: +*************************** Appendix F Python Embedding -=========================== +*************************** Introduction -____________ +============ MET includes the ability to embed Python to a limited degree. Users may use Python scripts and whatever associated Python packages they wish in order to prepare 2D gridded data fields, point observations, and matched pairs as input to the MET tools. We fully expect that this degree of embedding will increase in the future. In addition, plans are in place to extend Python with MET in upcoming releases, allowing users to invoke MET tools directly from their Python script. While MET version 8.0 was built on Python 2.x, MET versions 9.0 and beyond are built on Python 3.6+. Compiling Python Support -________________________ +======================== In order to use Python embedding, the user's local Python installation must have the C-language Python header files and libraries. Sometimes when Python is installed locally, these header files and libraries are deleted at the end of the installation process, leaving only the binary executable and run-time shared object files. But the Python header files and libraries must be present to compile support in MET for Python embedding. Assuming the requisite Python files are present, and that Python embedding is enabled when building MET (which is done by passing the **--enable-python** option to the **configure** command line), the MET C++ code will use these in the compilation process to link directly to the Python libraries. @@ -25,7 +26,7 @@ Make sure that these are set as environment variables or that you have included MET_PYTHON_EXE -______________ +============== When Python embedding support is compiled, MET instantiates the Python interpreter directly. However, for users of highly configurable Conda environments, the Python instance set at compilation time may not be sufficient. Users may want to switch between Conda environments for which different packages are available. MET version 9.0 has been enhanced to address this need. @@ -48,7 +49,7 @@ With this approach, users should be able to execute Python scripts in their own .. _pyembed-2d-data: Python Embedding for 2D data -____________________________ +============================ We now describe how to write Python scripts so that the MET tools may extract 2D gridded data fields from them. Currently, MET offers two ways to interact with Python scripts: by using NumPy N-dimensional arrays (ndarrays) or by using Xarray DataArrays. The interface to be used (NumPy or Xarray) is specified on the command line (more on this later). The user's scripts can use any Python libraries that are supported by the local Python installation, or any personal or institutional libraries or code that are desired in order to implement the Python script, so long as the data has been loaded into either a NumPy ndarray or an Xarray DataArray by the end of the script. This offers advantages when using data file formats that MET does not directly support. If there is Python code to read the data format, the user can use those tools to read the data, and then copy the data into a NumPy ndarray or an Xarray DataArray. MET can then ingest the data via the Python script. Note that whether a NumPy ndarray or an Xarray DataArray is used, the data should be stored as double precision floating point numbers. Using different data types, such as integers or single precision floating point numbers, will lead to unexpected results in MET. @@ -236,7 +237,7 @@ The Ensemble-Stat, Series-Analysis, and MTD tools support the use of file lists -title "Python enabled plot_data_plane" Python Embedding for Point Observations -_______________________________________ +======================================= The ASCII2NC tool supports the "-format python" option. With this option, point observations may be passed as input. An example of this is provided in :numref:`ascii2nc-pyembed`. That example uses the **read_ascii_point.py** sample script which is included with the MET code. It reads ASCII data in MET's 11-column point observation format and stores it in a Pandas dataframe to be read by the ASCII2NC tool with Python. @@ -248,7 +249,7 @@ The **read_ascii_point.py** sample script can be found in: • `MET GitHub repository `_ in *met/scripts/python*. Python Embedding for MPR data -_____________________________ +============================= The Stat-Analysis tool supports the "-lookin python" option. With this option, matched pair (MPR) data may be passed as input. An example of this is provided in :numref:`StA-pyembed`. That example uses the **read_ascii_mpr.py** sample script which is included with the MET code. It reads MPR data and stores it in a Pandas dataframe to be read by the Stat-Analysis tool with Python. @@ -260,7 +261,7 @@ The **read_ascii_mpr.py** sample script can be found in: Python Embedding for Point Observations as input -________________________________________________ +================================================ The point2grid, plot_point_obs, ensemble_stat, and point_stat tools use MET point observation NetCDF. They support the python embedding by the prefix 'PYTHON_NUMPY=" and followed by a python script name instead of the MET point observastion NetCDF filename. The customized python script is expected to extend MET_BASE/python/met_point_obs.py and to produce the python variable, **met_point_data**, which is the dictionary of the MET point observation data. They are defined at MET_BASE/python/met_point_obs.py. diff --git a/met/docs/Users_Guide/appendixG.rst b/met/docs/Users_Guide/appendixG.rst index 8c46641ced..c2811fe4fa 100644 --- a/met/docs/Users_Guide/appendixG.rst +++ b/met/docs/Users_Guide/appendixG.rst @@ -1,7 +1,8 @@ .. _appendixG: +**************************************** Appendix G Vectors and Vector Statistics -======================================== +**************************************** In this appendix, we discuss some basic properties of vectors, concentrating on the two-dimensional case. To keep the discussion simple, we will assume we are using a Cartesian coordinate system. diff --git a/met/docs/Users_Guide/config_options_tc.rst b/met/docs/Users_Guide/config_options_tc.rst index b9542a878d..17b1c266c4 100644 --- a/met/docs/Users_Guide/config_options_tc.rst +++ b/met/docs/Users_Guide/config_options_tc.rst @@ -1,16 +1,16 @@ .. _config_options_tc: +************************************** Tropical Cyclone Configuration Options -====================================== +************************************** See :numref:`config_options` for a description of the configuration file syntax. Configuration settings common to multiple tools -_______________________________________________ +=============================================== -.. _storm_id_1: - -:ref:`storm_id ` +storm_id_1 +---------- Specify a comma-separated list of storm id's to be used: @@ -30,9 +30,8 @@ This may also be set using basin, cyclone, and timing information below. storm_id = []; -.. _basin: - -:ref:`basin ` +basin +----- Specify a comma-separated list of basins to be used. Expected format is a 2-letter basin identifier. An empty list indicates that all should be used. @@ -50,9 +49,8 @@ For example: basin = []; -.. _cyclone: - -:ref:`cyclone ` +cyclone +------- Specify a comma-separated list of cyclone numbers (01-99) to be used. An empty list indicates that all should be used. @@ -66,9 +64,8 @@ For example: cyclone = []; -.. _storm_name_1: - -:ref:`storm_name ` +storm_name_1 +------------ Specify a comma-separated list of storm names to be used. An empty list indicates that all should be used. @@ -82,9 +79,8 @@ For example: storm_name = []; -.. _init_beg end inc exc: - -:ref:`init_beg, init_end, init_inc, init_exc ` +init_beg end inc exc +-------------------- Specify a model initialization time window in YYYYMMDD[_HH[MMSS]] format or provide a list of specific initialization times to include (inc) @@ -109,9 +105,8 @@ For example: init_inc = []; init_exc = []; -.. _valid_beg end inc exc_1: - - ref:`valid_beg, valid_end, valid_inc, valid_exc ` +valid_beg end inc exc_1 +----------------------- Specify a model valid time window YYYYMMDD[_HH[MMSS]] format or provide a list of specific valid times to include (inc) or exclude (exc). If a time @@ -138,9 +133,8 @@ For example: valid_inc = []; valid_exc = []; -.. _init_hour_1: - -:ref:`init_hour ` +init_hour_1 +----------- Specify a comma-separated list of model initialization hours to be used in HH[MMSS] format. An empty list indicates that all hours should be used. @@ -154,9 +148,8 @@ For example: init_hour = []; -.. _lead_req: - -:ref:`lead_req ` +lead_req +-------- Specify the required lead time in HH[MMSS] format. Tracks that contain all of these required times will be @@ -169,9 +162,8 @@ all lead times will be used. lead_req = []; -.. _init_mask, valid_mask: - -:ref:`init_mask, valid_mask ` +init_mask, valid_mask +--------------------- Specify lat/lon polylines defining masking regions to be applied. Tracks whose initial location falls within init_mask will be used. @@ -187,9 +179,8 @@ For example: init_mask = ""; valid_mask = ""; -.. _version: - -:ref:`version ` +version +------- Indicate the version number for the contents of this configuration file. The value should generally not be modified. @@ -200,15 +191,14 @@ The value should generally not be modified. Settings specific to individual tools -_____________________________________ +===================================== TCPairsConfig_default -~~~~~~~~~~~~~~~~~~~~~ +--------------------- -.. _model_1: - -:ref:`model ` +model_1 +^^^^^^^ The "model" entry specifies an array of model names to be verified. If verifying multiple models, choose descriptive model names (no whitespace) @@ -223,9 +213,8 @@ For example: model = []; -.. _check_dup: - -:ref:`check_dup ` +check_dup +^^^^^^^^^ Specify whether the code should check for duplicate ATCF lines when building tracks. Setting this to FALSE makes the parsing of tracks quicker. @@ -239,9 +228,8 @@ For example: check_dup = FALSE; -.. _interp12: - -:ref:`interp12 ` +interp12 +^^^^^^^^ Specify whether special processing should be performed for interpolated model names ending in 'I' (e.g. AHWI). Search for corresponding tracks whose model @@ -259,9 +247,8 @@ name ends in '2' (e.g. AHW2) and apply the following logic: interp12 = REPLACE; -.. _consensus: - -:ref:`consensus ` +consensus +^^^^^^^^^ Specify how consensus forecasts should be defined: @@ -288,9 +275,8 @@ For example: consensus = []; -.. _lag_time: - -:ref:`lag_time ` +lag_time +^^^^^^^^ Specify a comma-separated list of forecast lag times to be used in HH[MMSS] format. For each ADECK track identified, a lagged track will be derived @@ -306,9 +292,8 @@ For example: lag_time = []; -.. _best: - -:ref:`best_technique, best_baseline, oper_technique, oper_baseline ` +best +^^^^ Specify comma-separated lists of CLIPER/SHIFOR baseline forecasts to be derived from the BEST and operational tracks, as defined by the @@ -333,9 +318,8 @@ For example: oper_baseline = []; -.. _anly_track: - -:ref:`anly_track ` +anly_track +^^^^^^^^^^ Analysis tracks consist of multiple track points with a lead time of zero for the same storm. An analysis track may be generated by running model @@ -353,9 +337,8 @@ For example: anly_track = BDECK; -.. _match_points: - -:ref:`match_points ` +match_points +^^^^^^^^^^^^ Specify whether only those track points common to both the ADECK and BDECK tracks should be written out. @@ -371,9 +354,8 @@ For example: match_points = FALSE; -.. _dland_file: - -:ref:`dland_file ` +dland_file +^^^^^^^^^^ Specify the NetCDF output of the gen_dland tool containing a gridded representation of the minimum distance to land. @@ -384,9 +366,8 @@ representation of the minimum distance to land. dland_file = "MET_BASE/tc_data/dland_nw_hem_tenth_degree.nc"; -.. _watch_warn: - -:ref:`watch_warn ` +watch_warn +^^^^^^^^^^ Specify watch/warning information. Specify an ASCII file containing watch/warning information to be used. At each track point, the most severe @@ -404,9 +385,8 @@ occurring 4 hours (-14400 second) prior to the watch/warning time. } -.. _basin_map: - -:ref:`basin_map ` +basin_map +^^^^^^^^^ The basin_map entry defines a mapping of input names to output values. Whenever the basin string matches "key" in the input ATCF files, it is @@ -443,11 +423,10 @@ parameter will result in missed matches. ]; TCStatConfig_default -____________________ - -.. _amodel, bmodel: +==================== -:ref:`amodel, bmodel ` +amodel, bmodel +-------------- Stratify by the AMODEL or BMODEL columns. Specify comma-separated lists of model names to be used for all analyses @@ -465,9 +444,8 @@ For example: amodel = []; bmodel = []; -.. _valid_beg end inc exc: - - ref:`valid_beg, valid_end, valid_inc, valid_exc ` +valid_beg end inc exc +--------------------- Stratify by the VALID times. Define beginning and ending time windows in YYYYMMDD[_HH[MMSS]] @@ -490,9 +468,8 @@ For example: valid_inc = []; valid_exc = []; -.. _init valid_hour lead req: - -:ref:`init_hour, valid_hour, lead, lead_req ` +init valid_hour lead req +------------------------ Stratify by the initialization and valid hours and lead time. Specify a comma-separated list of initialization hours, @@ -516,9 +493,8 @@ For example: lead_req = []; -.. _line_type: - -:ref:`line_type ` +line_type +--------- Stratify by the LINE_TYPE column. May add using the "-line_type" job command option. @@ -532,9 +508,8 @@ For example: line_type = []; -.. _track_watch_warn: - -:ref:`track_watch_warn ` +track_watch_warn +---------------- Stratify by checking the watch/warning status for each track point common to both the ADECK and BDECK tracks. If the watch/warning status @@ -554,9 +529,8 @@ For example: track_watch_warn = []; -.. _column_thresh_name_and_val: - -:ref:`column_thresh_name, column_thresh_val ` +column_thresh_name_and_val +-------------------------- Stratify by applying thresholds to numeric data columns. Specify a comma-separated list of columns names and thresholds @@ -574,9 +548,8 @@ For example: column_thresh_name = []; column_thresh_val = []; -.. _column_str_name, column_str_val: - -:ref:`column_str_name, column_str_val ` +column_str_name, column_str_val +------------------------------- Stratify by performing string matching on non-numeric data columns. Specify a comma-separated list of columns names and values @@ -594,9 +567,8 @@ For example: column_str_name = []; column_str_val = []; -.. _column_str_name val: - -:ref:`column_str_exc_name, column_str_exc_val ` +column_str_name val +------------------- Stratify by performing string matching on non-numeric data columns. Specify a comma-separated list of columns names and values @@ -614,9 +586,8 @@ For example: column_str_exc_name = []; column_str_exc_val = []; -.. _init_thresh_name, init_thresh_val: - -:ref:`init_thresh_name, init_thresh_val ` +init_thresh_name, init_thresh_val +--------------------------------- Just like the column_thresh options above, but apply the threshold only when lead = 0. If lead = 0 value does not meet the threshold, discard @@ -633,9 +604,9 @@ For example: init_thresh_name = []; init_thresh_val = []; -.. _init_str_name, init_str_val: - -:ref:`init_str_name, init_str_val ` + +init_str_name, init_str_val +--------------------------- Just like the column_str options above, but apply the string matching only when lead = 0. If lead = 0 string does not match, discard the entire track. @@ -652,9 +623,8 @@ For example: init_str_name = []; init_str_val = []; -.. _init_str_exc_name and _exc_val: - -:ref:`init_str_exc_name, init_str_exc_val ` +init_str_exc_name and _exc_val +------------------------------ Just like the column_str_exc options above, but apply the string matching only when lead = 0. If lead = 0 string does match, discard the entire track. @@ -671,9 +641,8 @@ For example: init_str_exc_name = []; init_str_exc_val = []; -.. _water_only: - -:ref:`water_only ` +water_only +---------- Stratify by the ADECK and BDECK distances to land. Once either the ADECK or BDECK track encounters land, discard the remainder of the track. @@ -687,9 +656,8 @@ For example: water_only = FALSE; -.. _rirw: - -:ref:`rirw ` +rirw +---- Specify whether only those track points for which rapid intensification or weakening of the maximum wind speed occurred in the previous time @@ -721,9 +689,8 @@ May modify using the following job command options: bdeck = adeck; Copy settings to the BDECK or specify different logic. } -.. _landfall beg end: - -:ref:`landfall, landfall_beg, landfall_end ` +landfall beg end +---------------- Specify whether only those track points occurring near landfall should be retained, and define the landfall retention window as a time string in HH[MMSS] @@ -754,9 +721,8 @@ For example: landfall_beg = "-24"; landfall_end = "00"; -.. _event_equal: - -:ref:`event_equal ` +event_equal +----------- Specify whether only those cases common to all models in the dataset should be retained. May modify using the "-event_equal" job command option. @@ -771,9 +737,8 @@ For example: event_equal = FALSE; -.. _event_equal_lead: - -:ref:`event_equal_lead ` +event_equal_lead +---------------- Specify lead times that must be present for a track to be included in the event equalization logic. @@ -783,9 +748,8 @@ event equalization logic. event_equal_lead = [ "12", "24", "36" ]; -.. _out_int_mask: - -:ref:`out_int_mask ` +out_int_mask +------------ Apply polyline masking logic to the location of the ADECK track at the initialization time. If it falls outside the mask, discard the entire track. @@ -801,9 +765,8 @@ For example: out_init_mask = ""; -.. _out_valid_mask: - -:ref:`out_valid_mask ` +out_valid_mask +-------------- Apply polyline masking logic to the location of the ADECK track at the valid time. If it falls outside the mask, discard only the current track @@ -818,9 +781,8 @@ For example: out_valid_mask = ""; -.. _job: - -:ref:`job ` +job +--- The "jobs" entry is an array of TCStat jobs to be performed. Each element in the array contains the specifications for a single analysis @@ -937,7 +899,8 @@ Where "job_name" is set to one of the following: Default search time window is 0 0, requiring exact match -rirw_time or -rirw_time_adeck and -rirw_time_bdeck to override defaults -rirw_exact or -rirw_exact_adeck and -rirw_exact_bdeck to override defaults - -rirw_thresh or -rirw_thresh_adeck and -rirw_thresh_bdeck to override defaults + -rirw_thresh or -rirw_thresh_adeck and -rirw_thresh_bdeck to override + defaults -by column_name to specify case information -out_alpha to override default alpha value -out_line_type to specify output line types (CTC, CTS, and MPR) @@ -1004,11 +967,10 @@ Where "job_name" is set to one of the following: jobs = []; TCGenConfig_default -___________________ +=================== -.. _int_freq: - -:ref:`int_freq ` +int_freq +-------- Model initialization frequency in hours, starting at 0. @@ -1016,9 +978,8 @@ Model initialization frequency in hours, starting at 0. init_freq = 6; -.. _lead_window: - -:ref:`lead_window ` +lead_window +----------- Lead times in hours to be searched for genesis events. @@ -1030,9 +991,8 @@ Lead times in hours to be searched for genesis events. end = 120; } -.. _min_duration: - -:ref:`min_duration ` +min_duration +------------ Minimum track duration for genesis event in hours. @@ -1040,9 +1000,8 @@ Minimum track duration for genesis event in hours. min_duration = 12; -.. _fcst_genesis: - -:ref:`fcst_genesis ` +fcst_genesis +------------ Forecast genesis event criteria. Defined as tracks reaching the specified intensity category, maximum wind speed threshold, and minimum sea-level @@ -1057,9 +1016,8 @@ track point where all of these criteria are met. mslp_thresh = NA; } -.. _best_genesis: - -:ref:`best_genesis ` +best_genesis +------------ BEST track genesis event criteria. Defined as tracks reaching the specified intensity category, maximum wind speed threshold, and minimum sea-level @@ -1075,9 +1033,8 @@ first track point where all of these criteria are met. mslp_thresh = NA; } -.. _oper_genesis: - -:ref:`oper_genesis ` +oper_genesis +------------ Operational track genesis event criteria. Defined as tracks reaching the specified intensity category, maximum wind speed threshold, and minimum @@ -1094,11 +1051,10 @@ time of the first track point where all of these criteria are met. } Track filtering options which may be specified separately in each filter array entry -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------------------------------------------ -.. _filter: - -:ref:`filter ` +filter +^^^^^^ Filter is an array of dictionaries containing the track filtering options listed below. If empty, a single filter is defined using the top-level @@ -1108,9 +1064,8 @@ settings. filter = []; -.. _desc: - -:ref:`desc ` +desc +^^^^ Description written to output DESC column @@ -1118,9 +1073,8 @@ Description written to output DESC column desc = "NA"; -.. _model_2: - -:ref:`model ` +model_2 +^^^^^^^ Forecast ATCF ID's If empty, all ATCF ID's found will be processed. @@ -1131,9 +1085,8 @@ Statistics will be generated separately for each ATCF ID. model = []; -.. _storm_id_2: - -:ref:`storm_id ` +storm_id_2 +^^^^^^^^^^ BEST and operational track storm identifiers @@ -1141,9 +1094,8 @@ BEST and operational track storm identifiers storm_id = []; -.. _storm_name_2: - -:ref:`storm_name ` +storm_name_2 +^^^^^^^^^^^^ BEST and operational track storm names @@ -1151,9 +1103,8 @@ BEST and operational track storm names storm_name = []; -.. _init_beg, init_end2: - -:ref:`init_beg, init_end ` +init_beg, init_end2 +^^^^^^^^^^^^^^^^^^^ Forecast and operational initialization time window @@ -1162,9 +1113,8 @@ Forecast and operational initialization time window init_beg = ""; init_end = ""; -.. _valid_beg, valid_end_2: - -:ref:`valid_beg, valid_end ` +valid_beg, valid_end_2 +^^^^^^^^^^^^^^^^^^^^^^ Forecast, BEST, and operational valid time window @@ -1173,9 +1123,8 @@ Forecast, BEST, and operational valid time window valid_beg = ""; valid_end = ""; -.. _init_hour_2: - -:ref:`init_hour ` +init_hour_2 +^^^^^^^^^^^ Forecast and operational initialization hours @@ -1183,9 +1132,8 @@ Forecast and operational initialization hours init_hour = []; -.. _lead: - -:ref:`lead ` +lead +^^^^ Forecast and operational lead times in hours @@ -1193,9 +1141,8 @@ Forecast and operational lead times in hours lead = []; -.. _vx_mask: - -:ref:`vx_mask ` +vx_mask +^^^^^^^ Spatial masking region (path to gridded data file or polyline file) @@ -1203,9 +1150,8 @@ Spatial masking region (path to gridded data file or polyline file) vx_mask = ""; -.. _dland_thresh: - -:ref:`dland_thresh ` +dland_thresh +^^^^^^^^^^^^ Distance to land threshold @@ -1213,9 +1159,8 @@ Distance to land threshold dland_thresh = NA; -.. _genesis_window: - -:ref:`genesis_window ` +genesis_window +^^^^^^^^^^^^^^ Genesis matching time window, in hours relative to the forecast genesis time @@ -1226,9 +1171,8 @@ Genesis matching time window, in hours relative to the forecast genesis time end = 24; } -.. _genesis_radius: - -:ref:`genesis_radius ` +genesis_radius +^^^^^^^^^^^^^^ Genesis matching search radius in km. @@ -1237,11 +1181,10 @@ Genesis matching search radius in km. genesis_radius = 300; Global settings -_______________ - -.. _ci_alpha: +=============== -:ref:`ci_alpha ` +ci_alpha +-------- Confidence interval alpha value @@ -1249,9 +1192,8 @@ Confidence interval alpha value ci_alpha = 0.05; -.. _output_flag: - -:ref:`output_flag ` +output_flag +----------- Statistical output types diff --git a/met/docs/Users_Guide/data_io.rst b/met/docs/Users_Guide/data_io.rst index dcca8763f9..6c05461f43 100644 --- a/met/docs/Users_Guide/data_io.rst +++ b/met/docs/Users_Guide/data_io.rst @@ -1,14 +1,15 @@ .. _data_io: +************ MET Data I/O -============ +************ Data must often be preprocessed prior to using it for verification. Several MET tools exist for this purpose. In addition to preprocessing observations, some plotting utilities for data checking are also provided and described at the end of this section. Both the input and output file formats are described in this section. :numref:`Input data formats` and :numref:`Intermediate data formats` are primarily concerned with re-formatting input files into the intermediate files required by some MET modules. These steps are represented by the first three columns in the MET flowchart depicted in :numref:`overview-figure`. Output data formats are described in :numref:`Output data formats`. Common configuration files options are described in :numref:`Configuration File Details`. Description of software modules used to reformat the data may now be found in :numref:`reformat_point` and :numref:`reformat_grid`. .. _Input data formats: Input data formats -__________________ +================== The MET package can handle multiple gridded input data formats: GRIB version 1, GRIB version 2, and NetCDF files following the Climate and Forecast (CF) conventions, containing WRF output post-processed using wrf_interp, or produced by the MET tools themselves. MET supports standard NCEP, USAF, UKMet Office and ECMWF GRIB tables along with custom, user-defined GRIB tables and the extended PDS including ensemble member metadata. See :numref:`Configuration File Details` for more information. Point observation files may be supplied in either PrepBUFR, ASCII, or MADIS format. Note that MET does not require the Unified Post-Processor to be used, but does require that the input GRIB data be on a standard, de-staggered grid on pressure or regular levels in the vertical. While the Grid-Stat, Wavelet-Stat, MODE, and MTD tools can be run on a gridded field at virtually any level, the Point-Stat tool can only be used to verify forecasts at the surface or on pressure or height levels. MET does not interpolate between native model vertical levels. @@ -19,7 +20,7 @@ Input point observation files in PrepBUFR format are available through NCEP. The Tropical cyclone forecasts and observations are typically provided in a specific ATCF (Automated Tropical Cyclone Forecasting) ASCII format, in A-deck, B-deck, and E-deck files. Requirements for CF Compliant NetCDF -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------ The MET tools use following attributes and variables for input CF Compliant NetCDF data. @@ -124,7 +125,7 @@ MET gets the valid time from the time variable and the "forecast_reference_time" "degreesE" Performance with NetCDF input data -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------- There is no limitation on the NetCDF file size. The size of the data variables matters more than the file size. The NetCDF API loads the metadata first upon opening the NetCDF file. It's similar for accessing data variables. There are two API calls: getting the metadata and getting the actual data. The memory is allocated and consumed at the second API call (getting the actual data). @@ -133,14 +134,14 @@ The dimensions of the data variables matter. MET requests the NetCDF data needs .. _Intermediate data formats: Intermediate data formats -_________________________ +========================= MET uses NetCDF as an intermediate file format. The MET tools which write gridded output files write to a common gridded NetCDF file format. The MET tools which write point output files write to a common point observation NetCDF file format. .. _Output data formats: Output data formats -___________________ +=================== The MET package currently produces output in the following basic file formats: STAT files, ASCII files, NetCDF files, PostScript plots, and png plots from the Plot-Mode-Field utility. @@ -161,7 +162,7 @@ Users can use the optional plotting utilities Plot-Data-Plane, Plot-Point-Obs, a .. _Data format summary: Data format summary -___________________ +=================== The following is a summary of the input and output formats for each of the tools currently in MET. The output listed is the maximum number of possible output files. Generally, the type of output files generated can be controlled by the configuration files and/or the command line options: @@ -367,7 +368,7 @@ The following is a summary of the input and output formats for each of the tools .. _Configuration File Details: Configuration File Details -__________________________ +========================== Part of the strength of MET is the leveraging of capability across tools. There are several configuration options that are common to many of the tools. diff --git a/met/docs/Users_Guide/ensemble-stat.rst b/met/docs/Users_Guide/ensemble-stat.rst index fd1dbf535a..ca449cdd9c 100644 --- a/met/docs/Users_Guide/ensemble-stat.rst +++ b/met/docs/Users_Guide/ensemble-stat.rst @@ -1,18 +1,19 @@ .. _ensemble-stat: +****************** Ensemble-Stat Tool -================== +****************** Introduction -____________ +============ The Ensemble-Stat tool may be run to create simple ensemble forecasts (mean, probability, spread, etc) from a set of several forecast model files to be used by the MET statistics tools. If observations are also included, ensemble statistics such as rank histograms, probability integral transform histograms, spread/skill variance, relative position and continuous ranked probability score are produced. Climatological mean and standard deviation data may also be provided, and will be used as a reference forecast in several of the output statistics. Finally, observation error perturbations can be included prior to calculation of statistics. Details about and equations for the statistics produced for ensembles are given in :numref:`Appendix C, Section %s `. Scientific and statistical aspects -__________________________________ +================================== Ensemble forecasts derived from a set of deterministic ensemble members -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------------------------------------------------- Ensemble forecasts are often created as a set of deterministic forecasts. The ensemble members are rarely used separately. Instead, they can be combined in various ways to produce a forecast. MET can combine the ensemble members into some type of summary forecast according to user specifications. Ensemble means are the most common, and can be paired with the ensemble variance or spread. Maximum, minimum and other summary values are also available, with details in the practical information section. @@ -23,7 +24,7 @@ The neighborhood ensemble probability (NEP) and neighborhood maximum ensemble pr The Ensemble-Stat tool writes the gridded relative frequencies, NEP, and NMEP fields to a NetCDF output file. Probabilistic verification methods can then be applied to those fields by evaluating them with the Grid-Stat and/or Point-Stat tools. Ensemble statistics -~~~~~~~~~~~~~~~~~~~ +------------------- Rank histograms and probability integral transform (PIT) histograms are used to determine if the distribution of ensemble values is the same as the distribution of observed values for any forecast field (:ref:`Hamill, 2001 `). The rank histogram is a tally of the rank of the observed value when placed in order with each of the ensemble values from the same location. If the distributions are identical, then the rank of the observation will be uniformly distributed. In other words, it will fall among the ensemble members randomly in equal likelihood. The PIT histogram applies this same concept, but transforms the actual rank into a probability to facilitate ensembles of differing sizes or with missing members. @@ -34,7 +35,7 @@ The relative position (RELP) is a count of the number of times each ensemble mem The ranked probability score (RPS) is included in the Ranked Probability Score (RPS) line type. It is the mean of the Brier scores computed from ensemble probabilities derived for each probability category threshold (prob_cat_thresh) specified in the configuration file. The continuous ranked probability score (CRPS) is the average the distance between the forecast (ensemble) cumulative distribution function and the observation cumulative distribution function. It is an analog of the Brier score, but for continuous forecast and observation fields. The CRPS statistic is computed using two methods: assuming a normal distribution defined by the ensemble mean and spread (:ref:`Gneiting et al., 2004 `) and using the empirical ensemble distribution (:ref:`Hersbach, 2000 `). The CRPS statistic is included in the Ensemble Continuous Statistics (ECNT) line type, along with other statistics quantifying the ensemble spread and ensemble mean skill. Climatology data -~~~~~~~~~~~~~~~~ +---------------- The Ensemble-Stat output includes at least three statistics computed relative to external climatology data. The climatology is defined by mean and standard deviation fields, and typically both are required in the computation of ensemble skill score statistics. MET assumes that the climatology follows a normal distribution, defined by the mean and standard deviation at each point. @@ -43,7 +44,7 @@ When computing the CRPS skill score for (:ref:`Gneiting et al., 2004 `. can be replicated using the appropriate options. The user selects a distribution for the observation error, along with parameters for that distribution. Rescaling and bias correction can also be specified prior to the perturbation. Random draws from the distribution can then be added to either, or both, of the forecast and observed fields, including ensemble members. Details about the effects of the choices on verification statistics should be considered, with many details provided in the literature (*e.g.* :ref:`Candille and Talagrand, 2008 `; :ref:`Saetra et al., 2004 `; :ref:`Santos and Ghelli, 2012 `). Generally, perturbation makes verification statistics better when applied to ensemble members, and worse when applied to the observations themselves. @@ -52,12 +53,12 @@ Normal and uniform are common choices for the observation error distribution. Th Observation errors differ according to instrument, temporal and spatial representation, and variable type. Unfortunately, many observation errors have not been examined or documented in the literature. Those that have usually lack information regarding their distributions and approximate parameters. Instead, a range or typical value of observation error is often reported and these are often used as an estimate of the standard deviation of some distribution. Where possible, it is recommended to use the appropriate type and size of perturbation for the observation to prevent spurious results. Practical Information -_____________________ +===================== This section contains information about configuring and running the Ensemble-Stat tool. The Ensemble-Stat tool creates or verifies gridded model data. For verification, this tool can accept either gridded or point observations. If provided, the climatology data files must be gridded. The input gridded model, observation, and climatology datasets must be on the same grid prior to calculation of any statistics, and in one of the MET supported gridded file formats. If gridded files are not on the same grid, MET will do the regridding for you if you specify the desired output grid. The point observations must be formatted as the NetCDF output of the point reformatting tools described in :numref:`reformat_point`. ensemble_stat usage -~~~~~~~~~~~~~~~~~~~ +------------------- The usage statement for the Ensemble Stat tool is shown below: @@ -125,7 +126,7 @@ An example of the ensemble_stat calling sequence is shown below: In this example, the Ensemble-Stat tool will process six forecast files specified in the file list into an ensemble forecast. Observations in both point and grid format will be included, and be used to compute ensemble statistics separately. Ensemble Stat will create a NetCDF file containing requested ensemble fields and an output STAT file. ensemble_stat configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------- The default configuration file for the Ensemble-Stat tool named **EnsembleStatConfig_default** can be found in the installed *share/met/config* directory. Another version is located in *scripts/config*. We encourage users to make a copy of these files prior to modifying their contents. Each configuration file (both the default and sample) contains many comments describing its contents. The contents of the configuration file are also described in the subsections below. @@ -455,7 +456,7 @@ Refer to the description of the **boot** entry in :numref:`config_options` for m ensemble_stat output -~~~~~~~~~~~~~~~~~~~~ +-------------------- ensemble_stat can produce output in STAT, ASCII, and NetCDF formats. The ASCII output duplicates the STAT output but has the data organized by line type. The output files are written to the default output directory or the directory specified by the -outdir command line option. diff --git a/met/docs/Users_Guide/gen-ens-prod.rst b/met/docs/Users_Guide/gen-ens-prod.rst index 6d6e562ee0..cd6f963b4f 100644 --- a/met/docs/Users_Guide/gen-ens-prod.rst +++ b/met/docs/Users_Guide/gen-ens-prod.rst @@ -1,20 +1,21 @@ .. _gen-ens-prod: +***************** Gen-Ens-Prod Tool -================= +***************** Introduction -____________ +============ The Gen-Ens-Prod tool generates simple ensemble products (mean, spread, probability, etc) from gridded ensemble member input files. While it processes model inputs, but it does not compare them to observations or compute statistics. However, the output products can be passed as input to the MET statistics tools for comparison against observations. Climatological mean and standard deviation data may also be provided to define thresholds based on the climatological distribution at each grid point. Note that this ensemble product generation step was provided by the Ensemble-Stat tool in earlier versions of MET. The Gen-Ens-Prod tool replaces and extends that functionality. Users are strongly encouraged to migrate ensemble product generation from Ensemble-Stat to Gen-Ens-Prod, as new features will only be added to Gen-Ens-Prod and the existing Ensemble-Stat functionality will be deprecated in a future version. Scientific and statistical aspects -__________________________________ +================================== Ensemble forecasts derived from a set of deterministic ensemble members -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------------------------------------------------- Ensemble forecasts are often created as a set of deterministic forecasts. The ensemble members are rarely used separately. Instead, they can be combined in various ways to produce a forecast. MET can combine the ensemble members into some type of summary forecast according to user specifications. Ensemble means are the most common, and can be paired with the ensemble variance or spread. Maximum, minimum and other summary values are also available, with details in the practical information section. @@ -27,17 +28,17 @@ The neighborhood ensemble probability (NEP) and neighborhood maximum ensemble pr The Gen-Ens-Prod tool writes the gridded relative frequencies, NEP, and NMEP fields to a NetCDF output file. Probabilistic verification methods can then be applied to those fields by evaluating them with the Grid-Stat and/or Point-Stat tools. Climatology data -~~~~~~~~~~~~~~~~ +---------------- The ensemble relative frequencies derived by Gen-Ens-Prod are computed by applying threshold(s) to the input ensemble member data. Those thresholds can be simple and remain constant over the entire domain (e.g. >0) or can be defined relative to the climatological distribution at each grid point (e.g. >CDP90, for exceeding the 90-th percentile of climatology). When using climatological distribution percentile (CDP) thresholds, the climatological mean and standard deviation must be provided in the configuration file. Practical Information -_____________________ +===================== This section contains information about configuring and running the Gen-Ens-Prod tool. The Gen-Ens-Prod tool writes a NetCDF output file containing the requested ensemble product fields for each input field specified. If provided, the climatology data files must be gridded. All input gridded model and climatology datasets must be on the same grid. However, users may leverage the automated regridding feature in MET if the desired output grid is specified in the configuration file. gen_ens_prod usage -~~~~~~~~~~~~~~~~~~~ +------------------ The usage statement for the Ensemble Stat tool is shown below: @@ -54,7 +55,7 @@ The usage statement for the Ensemble Stat tool is shown below: gen_ens_prod has three required arguments and accepts several optional ones. Required arguments gen_ens_prod -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------- 1. The **-ens file_1 ... file_n** option specifies the ensemble member file names. This argument is not required when ensemble files are specified in the **ens_file_list**, detailed below. @@ -65,7 +66,7 @@ Required arguments gen_ens_prod 4. The **-config file** option is a **GenEnsProdConfig** file containing the desired configuration settings. Optional arguments for gen_ens_prod -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------------------------- 4. The **-ctrl file** option specifies the input file for the ensemble control member. Data for this member is included in the computation of the ensemble mean, but excluded from the spread. The control file should not appear in the **-ens** list of ensemble member files (unless processing a single file that contains all ensemble members). @@ -85,7 +86,7 @@ An example of the gen_ens_prod calling sequence is shown below: In this example, the Gen-Ens-Prod tool derives products from the input ensemble members listed on the command line. gen_ens_prod configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- The default configuration file for the Gen-Ens-Prod tool named **GenEnsProdConfig_default** can be found in the installed *share/met/config* directory. Another version is located in *scripts/config*. We encourage users to make a copy of these files prior to modifying their contents. The contents of the configuration file are described in the subsections below. @@ -265,7 +266,7 @@ The **ensemble_flag** specifies which derived ensemble fields should be calculat 14. Climatological Distribution Percentile field for each CDP threshold specified gen_ens_prod output -~~~~~~~~~~~~~~~~~~~~ +------------------- The Gen-Ens-Prod tools writes a gridded NetCDF output file whose file name is specified using the -out command line option. The contents of that file depend on the contents of the **ens.field** array, the **ensemble_flag** options selected, and the presence of climatology data. The NetCDF variable names are self-describing and include the name/level of the field being processed, the type of ensemble product, and any relevant threshold information. If **nc_var_str** is defined for an **ens.field** array entry, that string is included in the corresponding NetCDF output variable names. diff --git a/met/docs/Users_Guide/grid-diag.rst b/met/docs/Users_Guide/grid-diag.rst index 22200ad0e2..96ec66375a 100644 --- a/met/docs/Users_Guide/grid-diag.rst +++ b/met/docs/Users_Guide/grid-diag.rst @@ -1,18 +1,19 @@ .. _grid-diag: +************** Grid-Diag Tool -============== +************** Introduction -____________ +============ The Grid-Diag tool creates histograms (probability distributions when normalized) for an arbitrary collection of data fields and levels. Joint histograms will be created for all possible pairs of variables. Masks can be used to subset the data fields spatially. The histograms are accumulated over a time series of input data files, similar to Series-Analysis. Practical information -_____________________ +===================== grid_diag usage -~~~~~~~~~~~~~~~ +--------------- The following sections describe the usage statement, required arguments, and optional arguments for **grid_diag**. @@ -53,7 +54,7 @@ Optional arguments for grid_diag 6. The **-compress level** option indicates the desired level of compression (deflate level) for NetCDF variables. The valid level is between 0 and 9. The value of "level" will override the default setting of 0 from the configuration file or the environment variable MET_NC_COMPRESS. Setting the compression level to 0 will make no compression for the NetCDF output. Lower number is for fast compression and higher number is for better compression. grid_diag configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------- The default configuration file for the Grid-Diag tool named **GridDiagConfig_default** can be found in the installed *share/met/config/* directory. It is encouraged for users to copy these default files before modifying their contents. The contents of the configuration file are described in the subsections below. @@ -96,7 +97,7 @@ The **name** and **level** entries in the **data** dictionary define the data to Grid-Diag prints a warning message if the actual range of data values falls outside the range defined for that variable in the configuration file. Any data values less than the configured range are counted in the first bin, while values greater than the configured range are counted in the last bin. grid_diag output file -~~~~~~~~~~~~~~~~~~~~~ +--------------------- The NetCDF file has a dimension for each of the specified data variable and level combinations, e.g. APCP_L0 and PWAT_L0. The bin minimum, midpoint, and maximum values are indicated with an _min, _mid, or _max appended to the variable/level. diff --git a/met/docs/Users_Guide/grid-stat.rst b/met/docs/Users_Guide/grid-stat.rst index b96ed7a724..2e0716f505 100644 --- a/met/docs/Users_Guide/grid-stat.rst +++ b/met/docs/Users_Guide/grid-stat.rst @@ -1,21 +1,22 @@ .. _grid-stat: +************** Grid-Stat Tool -============== +************** Introduction -____________ +============ The Grid-Stat tool provides verification statistics for a matched forecast and observation grid. All of the forecast grid points in the region of interest are matched to observation grid points on the same grid. All the matched grid points are used to compute the verification statistics. The Grid-Stat tool functions in much the same way as the Point-Stat tool, except that no interpolation is required because the forecasts and observations are on the same grid. However, the interpolation parameters may be used to perform a smoothing operation on the forecast and observation fields prior to verification. In addition to traditional verification approaches, the Grid-Stat tool includes Fourier decompositions, gradient statistics, distance metrics, and neighborhood methods, designed to examine forecast performance as a function of spatial scale. Scientific and statistical aspects of the Grid-Stat tool are briefly described in this section, followed by practical details regarding usage and output from the tool. Scientific and statistical aspects -__________________________________ +================================== Statistical measures -~~~~~~~~~~~~~~~~~~~~ +-------------------- The Grid-Stat tool computes a wide variety of verification statistics. Broadly speaking, these statistics can be subdivided into three types of statistics: measures for categorical variables, measures for continuous variables, and measures for probabilistic forecasts. Further, when a climatology file is included, reference statistics for the forecasts compared to the climatology can be calculated. These categories of measures are briefly described here; specific descriptions of all measures are provided in :numref:`Appendix C, Section %s `. Additional information can be found in :ref:`Wilks (2011) ` and :ref:`Jolliffe and Stephenson (2012) `, and on the Collaboration for Australian Weather and Climate Research Forecast Verification - `Issues, Methods and FAQ web page `_. @@ -53,17 +54,17 @@ Use of analysis fields for verification The Grid-Stat tool allows evaluation of model forecasts using model analysis fields. However, users are cautioned that an analysis field is not independent of its parent model; for this reason verification of model output using an analysis field from the same model is generally not recommended and is not likely to yield meaningful information about model performance. Statistical confidence intervals -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------- The confidence intervals for the Grid-Stat tool are the same as those provided for the Point-Stat tool except that the scores are based on pairing grid points with grid points so that there are likely more values for each field making any assumptions based on the central limit theorem more likely to be valid. However, it should be noted that spatial (and temporal) correlations are not presently taken into account in the confidence interval calculations. Therefore, confidence intervals reported may be somewhat too narrow (e.g., :ref:`Efron, 2007 `). See :numref:`Appendix D, Section %s ` for details regarding confidence intervals provided by MET. Grid weighting -~~~~~~~~~~~~~~ +-------------- When computing continuous statistics on a regular large scale or global latitude-longitude grid, weighting may be applied in order to compensate for the meridian convergence toward higher latitudes. Grid square area weighting or weighting based on the cosine of the latitude are two configuration options in both point-stat and grid-stat. See :numref:`config_options` for more information. Neighborhood methods -~~~~~~~~~~~~~~~~~~~~ +-------------------- MET also incorporates several neighborhood methods to give credit to forecasts that are close to the observations, but not necessarily exactly matched up in space. Also referred to as "fuzzy" verification methods, these methods do not just compare a single forecast at each grid point to a single observation at each grid point; they compare the forecasts and observations in a neighborhood surrounding the point of interest. With the neighborhood method, the user chooses a distance within which the forecast event can fall from the observed event and still be considered a hit. In MET this is implemented by defining a square search window around each grid point. Within the search window, the number of observed events is compared to the number of forecast events. In this way, credit is given to forecasts that are close to the observations without requiring a strict match between forecasted events and observed events at any particular grid point. The neighborhood methods allow the user to see how forecast skill varies with neighborhood size and can help determine the smallest neighborhood size that can be used to give sufficiently accurate forecasts. @@ -72,7 +73,7 @@ There are several ways to present the results of the neighborhood approaches, su The user must specify several parameters in the grid_stat configuration file to utilize the neighborhood approach, such as the interpolation method, size of the smoothing window, and required fraction of valid data points within the smoothing window. For FSS-specific results, the user must specify the size of the neighborhood window, the required fraction of valid data points within the window, and the fractional coverage threshold from which the contingency tables are defined. These parameters are described further in the practical information section below. Fourier Decomposition -~~~~~~~~~~~~~~~~~~~~~ +--------------------- The MET software will compute the full one-dimensional Fourier transform, then do a partial inverse transform based on the two user-defined wave numbers. These two wave numbers define a band pass filter in the Fourier domain. This process is conceptually similar to the operation of projecting onto subspace in linear algebra. If one were to sum up all possible wave numbers the result would be to simply reproduce the raw data. @@ -81,14 +82,14 @@ Decomposition via Fourier transform allows the user to evaluate the model separa Wavelets, and in particular the MET wavelet tool, can also be used to define a band pass filter (:ref:`Casati et al., 2004 `; :ref:`Weniger et al., 2016 `). Both the Fourier and wavelet methods can be used to look at different spatial scales. Gradient Statistics -~~~~~~~~~~~~~~~~~~~ +------------------- The S1 score has been in historical use for verification of forecasts, particularly for variables such as pressure and geopotential height. This score compares differences between adjacent grid points in the forecast and observed fields. When the adjacent points in both forecast and observed fields exhibit the same differences, the S1 score will be the perfect value of 0. Larger differences will result in a larger score. Differences are computed in both of the horizontal grid directions and is not a true mathematical gradient. Because the S1 score focuses on differences only, any bias in the forecast will not be measured. Further, the score depends on the domain and spacing of the grid, so can only be compared on forecasts with identical grids. Distance Maps -~~~~~~~~~~~~~ +------------- The following methods can all be computed efficiently by utilizing fast algorithms developed for calculating distance maps. A distance map results from calculating the shortest distance from every grid point, **s=(x,y)**, in the domain, **D**, to the nearest one-valued grid point. In each of the following, it is understood that they are calculated between event areas **A**, from one field and observation event areas **B** from another. If the measure is applied to a feature within a field, then the distance map is still calculated over the entire original domain. Some of the distance map statistics are computed over the entire distance map, while others use only parts of it. @@ -125,7 +126,7 @@ The statistics derived from these distance maps are described in :numref:`Append .. _grid-stat_gbeta: :math:`\beta` and :math:`G_\beta` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------- See :numref:`App_C-gbeta` for the :math:`G` and :math:`G_\beta` equations. @@ -146,14 +147,14 @@ In some cases, a user may be interested in a much higher threshold than :math:`2 Since :math:`G_\beta` is so sensitive to the choice of :math:`\beta`, which is defined relative to the number of points in the verification domain, :math:`G_\beta` is only computed for the full verification domain. :math:`G_\beta` is reported as a bad data value for any masking region subsets of the full verification domain. Practical information -_____________________ +===================== This section contains information about configuring and running the Grid-Stat tool. The Grid-Stat tool verifies gridded model data using gridded observations. The input gridded model and observation datasets must be in one of the MET supported file formats. The requirement of having all gridded fields using the same grid specification was removed in METv5.1. There is a regrid option in the configuration file that allows the user to define the grid upon which the scores will be computed. The gridded observation data may be a gridded analysis based on observations such as Stage II or Stage IV data for verifying accumulated precipitation, or a model analysis field may be used. The Grid-Stat tool provides the capability of verifying one or more model variables/levels using multiple thresholds for each model variable/level. The Grid-Stat tool performs no interpolation when the input model, observation, and climatology datasets must be on a common grid. MET will interpolate these files to a common grid if one is specified. The interpolation parameters may be used to perform a smoothing operation on the forecast field prior to verifying it to investigate how the scale of the forecast affects the verification statistics. The Grid-Stat tool computes a number of continuous statistics for the forecast minus observation differences, discrete statistics once the data have been thresholded, or statistics for probabilistic forecasts. All types of statistics can incorporate a climatological reference. grid_stat usage -~~~~~~~~~~~~~~~ +--------------- The usage statement for the Grid-Stat tool is listed below: @@ -217,7 +218,7 @@ In the second example, the Grid-Stat tool will verify the model data in the samp .. _grid_stat-configuration-file: grid_stat configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------- The default configuration file for the Grid-Stat tool, named **GridStatConfig_default**, can be found in the installed *share/met/config* directory. Other versions of the configuration file are included in *scripts/config*. We recommend that users make a copy of the default (or other) configuration file prior to modifying it. The contents are described in more detail below. @@ -457,7 +458,7 @@ The **nc_pairs_var_suffix** entry is similar to the **nc_pairs_var_name** entry. .. _grid_stat-output: grid_stat output -~~~~~~~~~~~~~~~~ +---------------- grid_stat produces output in STAT and, optionally, ASCII and NetCDF formats. The ASCII output duplicates the STAT output but has the data organized by line type. The output files are written to the default output directory or the directory specified by the -outdir command line option. diff --git a/met/docs/Users_Guide/gsi-tools.rst b/met/docs/Users_Guide/gsi-tools.rst index 019e5c3f7b..b5bde4f9a7 100644 --- a/met/docs/Users_Guide/gsi-tools.rst +++ b/met/docs/Users_Guide/gsi-tools.rst @@ -1,7 +1,8 @@ .. _gsi_tools: +********* GSI Tools -========= +********* Gridpoint Statistical Interpolation (GSI) diagnostic files are binary files written out from the data assimilation code before the first and after each outer loop. The files contain useful information about how a single observation was used in the analysis by providing details such as the innovation (O-B), observation values, observation error, adjusted observation error, and quality control information. @@ -12,12 +13,12 @@ When MET reads GSI diagnostic files, the innovation (O-B; generated prior to the MET includes two tools for processing GSI diagnostic files. The GSID2MPR tool reformats individual GSI diagnostic files into the MET matched pair (MPR) format, similar to the output of the Point-Stat tool. The GSIDENS2ORANK tool processes an ensemble of GSI diagnostic files and reformats them into the MET observation rank (ORANK) line type, similar to the output of the Ensemble-Stat tool. The output of both tools may be passed to the Stat-Analysis tool to compute a wide variety of continuous, categorical, and ensemble statistics. GSID2MPR tool -_____________ +============= This section describes how to run the GSID2MPR tool. The GSID2MPR tool reformats one or more GSI diagnostic files into an ASCII matched pair (MPR) format, similar to the MPR output of the Point-Stat tool. The output MPR data may be passed to the Stat-Analysis tool to compute a wide variety of continuous or categorical statistics. gsid2mpr usage -~~~~~~~~~~~~~~ +-------------- The usage statement for the GSID2MPR tool is shown below: @@ -71,7 +72,7 @@ An example of the gsid2mpr calling sequence is shown below: In this example, the GSID2MPR tool will process a single input file named **diag_conv_ges.mem001** file, set the output **MODEL** header column to **GSI_MEM001**, and write output to the **out** directory. The output file is named the same as the input file but a **.stat** suffix is added to indicate its format. gsid2mpr output -~~~~~~~~~~~~~~~ +--------------- The GSID2MPR tool performs a simple reformatting step and thus requires no configuration file. It can read both conventional and radiance binary GSI diagnostic files. Support for additional GSI diagnostic file type may be added in future releases. Conventional files are determined by the presence of the string **conv** in the filename. Files that are not conventional are assumed to contain radiance data. Multiple files of either type may be passed in a single call to the GSID2MPR tool. For each input file, an output file will be generated containing the corresponding matched pair data. @@ -243,12 +244,12 @@ An example of the Stat-Analysis calling sequence is shown below: In this example, the Stat-Analysis tool will read MPR lines from the input file named **diag_conv_ges.mem001.stat**, retain only those lines where the **FCST_VAR** column indicates temperature (**t**) and where the **ANLY_USE** column has a value of 1.0, and derive continuous statistics. GSIDENS2ORANK tool -__________________ +================== This section describes how to run the GSIDENS2ORANK tool. The GSIDENS2ORANK tool processes an ensemble of GSI diagnostic files and reformats them into the MET observation rank (ORANK) line type, similar to the output of the Ensemble-Stat tool. The ORANK line type contains ensemble matched pair information and is analogous to the MPR line type for a deterministic model. The output ORANK data may be passed to the Stat-Analysis tool to compute ensemble statistics. gsidens2orank usage -~~~~~~~~~~~~~~~~~~~ +------------------- The usage statement for the GSIDENS2ORANK tool is shown below: @@ -306,7 +307,7 @@ An example of the gsidens2orank calling sequence is shown below: In this example, the GSIDENS2ORANK tool will process all of the ensemble members whose file name **matches diag_conv_ges.mem\*,** write output to the file named **diag_conv_ges_ens_mean_orank.txt**, and populate the output **ENS_MEAN** column with the values found in the **diag_conv_ges.ensmean** file rather than computing the ensemble mean values from the ensemble members on the fly. gsidens2orank output -~~~~~~~~~~~~~~~~~~~~ +-------------------- The GSIDENS2ORANK tool performs a simple reformatting step and thus requires no configuration file. The multiple files passed to it are interpreted as members of the same ensemble. Therefore, each call to the tool processes exactly one ensemble. All input ensemble GSI diagnostic files must be of the same type. Mixing conventional and radiance files together will result in a runtime error. The GSIDENS2ORANK tool processes each ensemble member and keeps track of the observations it encounters. It constructs a list of the ensemble values corresponding to each observation and writes an output ORANK line listing the observation value, its rank, and all the ensemble values. The random number generator is used by the GSIDENS2ORANK tool to randomly assign a rank value in the case of ties. diff --git a/met/docs/Users_Guide/index.rst b/met/docs/Users_Guide/index.rst index 5ee52862e0..9c17bc0a19 100644 --- a/met/docs/Users_Guide/index.rst +++ b/met/docs/Users_Guide/index.rst @@ -1,6 +1,6 @@ -============ +############ User's Guide -============ +############ **Foreword: A note to MET users** diff --git a/met/docs/Users_Guide/installation.rst b/met/docs/Users_Guide/installation.rst index b4459fed0d..04ecd35b9f 100644 --- a/met/docs/Users_Guide/installation.rst +++ b/met/docs/Users_Guide/installation.rst @@ -1,22 +1,23 @@ .. _installation: +************************************* Software Installation/Getting Started -===================================== +************************************* Introduction -____________ +============ This section describes how to install the MET package. MET has been developed and tested on Linux operating systems. Support for additional platforms and compilers may be added in future releases. The MET package requires many external libraries to be available on the user's computer prior to installation. Required and recommended libraries, how to install MET, the MET directory structure, and sample cases are described in the following sections. -Supported architectures -_______________________ +Supported Architectures +======================= The MET package was developed on Debian Linux using the GNU compilers and the Portland Group (PGI) compilers. The MET package has also been built on several other Linux distributions using the GNU, PGI, and Intel compilers. Past versions of MET have also been ported to IBM machines using the IBM compilers, but we are currently unable to support this option as the development team lacks access to an IBM machine for testing. Other machines may be added to this list in future releases as they are tested. In particular, the goal is to support those architectures supported by the WRF model itself. The MET tools run on a single processor. Therefore, none of the utilities necessary for running WRF on multiple processors are necessary for running MET. Individual calls to the MET tools have relatively low computing and memory requirements. However users will likely be making many calls to the tools and passing those individual calls to several processors will accomplish the verification task more efficiently. -Programming languages -_____________________ +Programming Languages +===================== The MET package, including MET-TC, is written primarily in C/C++ in order to be compatible with an extensive verification code base in C/C++ already in existence. In addition, the object-based MODE and MODE-TD verification tools rely heavily on the object-oriented aspects of C++. Knowledge of C/C++ is not necessary to use the MET package. The MET package has been designed to be highly configurable through the use of ASCII configuration files, enabling a great deal of flexibility without the need for source code modifications. @@ -24,8 +25,8 @@ NCEP's BUFRLIB is written entirely in Fortran. The portion of MET that handles t The MET package is intended to be a tool for the modeling community to use and adapt. As users make upgrades and improvements to the tools, they are encouraged to offer those upgrades to the broader community by offering feedback to the developers. -Required compilers and scripting languages -__________________________________________ +Required Compilers and Acripting Languages +========================================== The MET package was developed and tested using the GNU g++/gfortran compilers and the Intel icc/ifort compilers. As additional compilers are successfully tested, they will be added to the list of supported platforms/compilers. @@ -37,8 +38,8 @@ In order to control the desired flow through MET, users are encouraged to run th .. _Install_Required-libraries-and: -Required libraries and optional utilities -_________________________________________ +Required Libraries and Optional U1tilities +========================================== Three external libraries are required for compiling/building MET and should be downloaded and installed before attempting to install MET. Additional external libraries required for building advanced features in MET are discussed in :numref:`Installation-of-required` : @@ -58,8 +59,8 @@ Two additional utilities are strongly recommended for use with MET: .. _Installation-of-required: -Installation of required libraries -__________________________________ +Installation of Required Libraries +================================== As described in :numref:`Install_Required-libraries-and`, some external libraries are required for building the MET: @@ -105,8 +106,8 @@ In the directions above, the static library file that is created will be named l .. _Installation-of-optional: -Installation of optional utilities -__________________________________ +Installation of Optional Utilities +================================== As described in the introduction to this section, two additional utilities are strongly recommended for use with MET. @@ -116,8 +117,8 @@ As described in the introduction to this section, two additional utilities are s .. _met_directory_structure: -MET directory structure -_______________________ +MET Directory Structure +======================= The top-level MET directory consists of Makefiles, configuration files, and several subdirectories. The top-level Makefile and configuration files control how the entire toolkit is built. Instructions for using these files to build MET can be found in :numref:`Install_Building-the-MET`. @@ -137,20 +138,20 @@ The *share/met/Rscripts* directory contains a handful of sample R scripts, inclu .. _Install_Building-the-MET: -Building the MET package -________________________ +Building the MET Package +======================== Building the MET package consists of three main steps: (1) install the required libraries, (2) configure the environment variables, and (3) configure and execute the build. Users can follow the instructions below or use a sample installation script. Users can find the script and its instructions under on the `Downloads `_ page of the MET website. Install the Required Libraries -______________________________ +============================== • Please refer to :numref:`Installation-of-required` and :numref:`Installation-of-optional` on how to install the required and optional libraries. • If installing the required and optional libraries in a non-standard location, the user may need to tell MET where to find them. This can be done by setting or adding to the LD_LIBRARY PATH to include the path to the library files. Set Environment Variables -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- The MET build uses environment variables to specify the locations of the needed external libraries. For each library, there is a set of three environment variables to describe the locations: $MET_, $MET_INC and $MET_LIB. @@ -188,8 +189,8 @@ The following environment variables should also be set: For ease of use, you should define these in your .cshrc or equivalent file. -Configure and execute the build -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Configure and Execute the Build +------------------------------- Example: To configure MET to install all of the available tools in the "bin" subdirectory of your current directory, you would use the following commands: @@ -301,7 +302,7 @@ regions to benefit from thread-parallel execution. Runtime environment variable Run the configure script with the **-help** argument to see the full list of configuration options. Make Targets -~~~~~~~~~~~~ +------------ The autoconf utility provides some standard make targets for the users. In MET, the following standard targets have been implemented and tested: @@ -319,8 +320,8 @@ MET also has the following non-standard targets: .. _Sample Test cases: -Sample test cases -_________________ +Sample Test Cases +================= Once the MET package has been built successfully, the user is encouraged to run the sample test scripts provided. They are run using make test in the top-level directory. Execute the following commands: diff --git a/met/docs/Users_Guide/masking.rst b/met/docs/Users_Guide/masking.rst index db695f97f1..1f03ff2bce 100644 --- a/met/docs/Users_Guide/masking.rst +++ b/met/docs/Users_Guide/masking.rst @@ -1,17 +1,18 @@ .. _masking: +******************************************* Regional Verification using Spatial Masking -=========================================== +******************************************* Verification over a particular region or area of interest may be performed using "masking". Defining a masking region is simply selecting the desired set of grid points to be used. The Gen-Vx-Mask tool automates this process and replaces the Gen-Poly-Mask and Gen-Circle-Mask tools from previous releases. It may be run to create a bitmap verification masking region to be used by many of the statistical tools. This tool enables the user to generate a masking region once for a domain and apply it to many cases. It has been enhanced to support additional types of masking region definition (e.g. tropical-cyclone track over water only). An iterative approach may be used to define complex areas by combining multiple masking regions together. Gen-Vx-Mask tool -________________ +================ The Gen-Vx-Mask tool may be run to create a bitmap verification masking region to be used by the MET statistics tools. This tool enables the user to generate a masking region once for a domain and apply it to many cases. While the MET statistics tools can define some masking regions on the fly using polylines, doing so can be slow, especially for complex polylines containing hundreds of vertices. Using the Gen-Vx-Mask tool to create a bitmap masking region before running the other MET tools will make them run more efficiently. gen_vx_mask usage -~~~~~~~~~~~~~~~~~ +----------------- The usage statement for the Gen-Vx-Mask tool is shown below: @@ -162,6 +163,6 @@ An example of the gen_vx_mask calling sequence is shown below: In this example, the Gen-Vx-Mask tool will read the ASCII Lat/Lon file named **CONUS.poly** and apply the default polyline masking method to the domain on which the data in the file **sample_fcst.grib** resides. It will create a NetCDF file containing a bitmap for the domain with a value of 1 for all grid points inside the CONUS polyline and a value of 0 for all grid points outside. It will write an output NetCDF file named **CONUS_poly.nc**. Feature-Relative Methods -________________________ +======================== -This section contains a description of several methods that may be used to perform feature-relative (or event -based) evaluation. The methodology pertains to examining the environment surrounding a particular feature or event such as a tropical, extra-tropical cyclone, convective cell, snow-band, etc. Several approaches are available for these types of investigations including applying masking described above (e.g. circle or box) or using the "FORCE" interpolation method in the regrid configuration option (see :numref:`config_options`). These methods generally require additional scripting, including potentially storm-track identification, outside of MET to be paired with the features of the MET tools. METplus may be used to execute this type of analysis. Please refer to the `METplus User's Guide `__. +This section contains a description of several methods that may be used to perform feature-relative (or event -based) evaluation. The methodology pertains to examining the environment surrounding a particular feature or event such as a tropical, extra-tropical cyclone, convective cell, snow-band, etc. Several approaches are available for these types of investigations including applying masking described above (e.g. circle or box) or using the "FORCE" interpolation method in the regrid configuration option (see :numref:`config_options`). These methods generally require additional scripting, including potentially storm-track identification, outside of MET to be paired with the features of the MET tools. METplus may be used to execute this type of analysis. Please refer to the `METplus User's Guide `_. diff --git a/met/docs/Users_Guide/met-tc_overview.rst b/met/docs/Users_Guide/met-tc_overview.rst index 859e3e2efa..41ad7e5fc0 100644 --- a/met/docs/Users_Guide/met-tc_overview.rst +++ b/met/docs/Users_Guide/met-tc_overview.rst @@ -1,24 +1,25 @@ .. _met-tc_overview: +*************** MET-TC Overview -=============== +*************** Introduction -____________ +============ The purpose of this User's Guide is to provide basic information to the users of the Model Evaluation Tools - Tropical Cyclone (MET-TC) to enable users to apply MET-TC to their tropical cyclone datasets and evaluation studies. MET-TC is intended for use with model forecasts run through a vortex tracking software or with operational model forecasts in Automated Tropical Cyclone Forecast (ATCF) file format. The following sections provide an overview of MET-TC and its components, as well as basic information on the software build. The required input, including file format and the MET-TC are discussed followed by a description of the TC-Dland tool, TC-Pairs, and TC-Stat tools. Each section covers the input, output and practical usage including a description of the configuration files. This is followed by a short overview of graphical utilities available within the MET-TC release. MET-TC components -_________________ +================= The MET tools used in the verification of Tropical Cyclones are referred to as MET-TC. These tools are shown across the bottom of the flowchart in :numref:`overview-figure`. The MET-TC tools are described in more detail in later sections. The TC-Dland tool is used to generate a gridded file that determines the location of coastlines and islands, and is used as input to the TC-Pairs tool to determine the distance from land of a particular track point. The TC-Pairs tool matches pairs of input model data and BEST track (or any reference forecast) and calculates position errors. The TC-Stat tool uses the TC-Pairs output to perform filter and summary jobs over the matched pair dataset. The TC-Gen tool performs a categorical analysis for tropical cyclone genesis forecasts. The TC-RMW tool performs a coordinate transformation of gridded model data, centered on the storm's location. The RMW-Analysis tool aggregates TC-RMW output across multiple cases. Input data format -_________________ +================= This section discusses the input and output file formats expected and produced by MET-TC. When discussing the input data, it is expected that users have run model output through vortex tracking software in order to obtain position and intensity information in Automated Tropical Cyclone Forecasting System (ATCF) file format. Best track and aids files in Automated Tropical Cyclone Forecasting System (ATCF) format (hereafter referred to as ATCF format) are necessary for model data input into the TC-Pairs tool. The ATCF format was first developed at the Naval Oceanographic and Atmospheric Research Laboratory (NRL), and is currently used for the National Hurricane Center (NHC) operations. ATCF format must be adhered to in order for the MET-TC tools to properly parse the input data. @@ -109,6 +110,6 @@ All operational model aids and the BEST can be obtained from the `NHC ftp server If a user has gridded model output, the model data must be run through a vortex tracking algorithm in order to obtain the ATCF-formatted input that MET-TC requires. Many vortex tracking algorithms have been developed in order to obtain basic position, maximum wind, and minimum sea level pressure information from model forecasts. One vortex tracking algorithm that is supported and freely available is the `GFDL vortex tracker package. `_ Output data format -__________________ +================== The MET package produces output in four basic file formats: STAT files, ASCII files, NetCDF files, and Postscript plots. The MET-TC tool produces output in TCSTAT, which stands for Tropical Cyclone - STAT. This output format consists of tabular ASCII data that can be easily read by many analysis tools and software packages, making the output from MET-TC very versatile. Like STAT, TCSTAT is a specialized ASCII format containing one record on each line. Currently, the only line type available in MET-TC is TCMPR (Tropical Cyclone Matched Pairs). As more line types are included in future releases, all line types will be included in a single TCSTAT file. MET-TC also outputs a NetCDF format file in the TC-Dland tool, as input to the TC-Pairs tool. diff --git a/met/docs/Users_Guide/mode-analysis.rst b/met/docs/Users_Guide/mode-analysis.rst index c641f8b8e6..c37db8ac8b 100644 --- a/met/docs/Users_Guide/mode-analysis.rst +++ b/met/docs/Users_Guide/mode-analysis.rst @@ -1,31 +1,32 @@ .. _mode-analysis: +****************** MODE-Analysis Tool -================== +****************** Introduction -____________ +============ Users may wish to summarize multiple ASCII files produced by MODE across many cases. The MODE output files contain many output columns making it very difficult to interpret the results by simply browsing the files. Furthermore, for particular applications some data fields in the MODE output files may not be of interest. The MODE-Analysis tool provides a simple way to compute basic summary statistics and filtering capabilities for these files. Users who are not proficient at writing scripts can use the tool directly, and even those using their own scripts can use this tool as a filter, to extract only the MODE output lines that are relevant for their application. .. _MODE_A-Scientific-and-statistical: Scientific and statistical aspects -__________________________________ +================================== The MODE-Analysis tool operates in two modes, called "summary" and "bycase". In summary mode, the user specifies on the command line the MODE output columns of interest as well as filtering criteria that determine which input lines should be used. For example, a user may be interested in forecast object areas, but only if the object was matched, and only if the object centroid is inside a particular region. The summary statistics generated for each specified column of data are the minimum, maximum, mean, standard deviation, and the 10th, 25th, 50th, 75th and 90th percentiles. In addition, the user may specify a "dump'" file: the individual MODE lines used to produce the statistics will be written to this file. This option provides the user with a filtering capability. The dump file will consist only of lines that match the specified criteria. The other option for operating the analysis tool is "bycase". Given initial and final values for forecast lead time, the tool will output, for each valid time in the interval, the matched area, unmatched area, and the number of forecast and observed objects that were matched or unmatched. For the areas, the user can specify forecast or observed objects, and also simple or cluster objects. A dump file may also be specified in this mode. Practical information -_____________________ +===================== The MODE-Analysis tool reads lines from MODE ASCII output files and applies filtering and computes basic statistics on the object attribute values. For each job type, filter parameters can be set to determine which MODE output lines are used. The following sections describe the mode_analysis usage statement, required arguments, and optional arguments. .. _mode_analysis-usage: mode_analysis usage -~~~~~~~~~~~~~~~~~~~ +------------------- The usage statement for the MODE-Analysis tool is shown below: @@ -587,13 +588,13 @@ This option prints the usage message. .. _mode_analysis-configuration-file: mode_analysis configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------- To use the MODE-Analysis tool, the user must un-comment the options in the configuration file to apply them and comment out unwanted options. The options in the configuration file for the MODE-Analysis tools are the same as the MODE command line options described in :numref:`mode_analysis-usage`. The parameters that are set in the configuration file either add to or override parameters that are set on the command line. For the "set string" and "set integer type" options enclosed in brackets, the values specified in the configuration file are added to any values set on the command line. For the "toggle" and "min/max type" options, the values specified in the configuration file override those set on the command line. mode_analysis output -~~~~~~~~~~~~~~~~~~~~ +-------------------- The output of the MODE-Analysis tool is a self-describing tabular format written to standard output. The length and contents of the table vary depending on whether **-summary** or **-bycase** is selected. The contents also change for **-summary** depending on the number of columns specified by the user. diff --git a/met/docs/Users_Guide/mode-td.rst b/met/docs/Users_Guide/mode-td.rst index b646410835..bbda4dc315 100644 --- a/met/docs/Users_Guide/mode-td.rst +++ b/met/docs/Users_Guide/mode-td.rst @@ -1,13 +1,14 @@ .. _mode-td: +********************* MODE Time Domain Tool -===================== +********************* Introduction -____________ +============ Motivation -~~~~~~~~~~ +---------- MODE Time Domain (MTD) is an extension of the MODE object-based approach to verification. In addition to incorporating spatial information, MTD utilizes the time dimension to get at temporal aspects of forecast verification. Since the two spatial dimensions of traditional meteorological forecasts are retained in addition to the time dimension, the method is inherently three dimensional. Given that, however, the overall methodology has deliberately been kept as similar as possible to that of traditional MODE. @@ -28,17 +29,17 @@ At first glance, the addition of a third dimension would seem to entail no diffi In this section, we will assume that the reader has a basic familiarity with traditional MODE, its internal operation, (convolution thresholding, fuzzy logic matching and merging) and its output. We will not review these things here. Instead, we will point out differences in MTD from the way traditional MODE does things when they come up. This release is a beta version of MTD, intended mostly to encourage users to experiment with it and give us feedback and suggestions to be used in a more robust MTD release in the future. Scientific and statistical aspects -__________________________________ +================================== Attributes -~~~~~~~~~~ +---------- Object attributes are, for the most part, calculated in much the same way in MTD as they are in MODE, although the fact that one of the dimensions is non-spatial introduces a few quirks. Several of the object attributes that traditional MODE calculates assume that distances, angles and areas can be calculated in grid coordinates via the usual Euclidean/Cartesian methods. That is no longer the case in spacetime, since there is no distance function (more precisely, no *metric*) there. Given two points in this spacetime, say :math:`(x_1, y_1, t_1)` and :math:`(x_2, y_2, t_2)`, there is no way to measure their separation with a single nonnegative number in a physically meaningful way. If all three of our dimensions were spatial, there would be no difficulties. This means that some care must be taken both in determining how to generalize the calculation of a geometric attribute to three-dimensional spacetime, and also in interpreting the attributes even in the case where the generalization is straightforward. Convolution -~~~~~~~~~~~ +----------- As in MODE, MTD applies a convolution filter to the raw data as a preliminary step in resolving the field into objects. The convolution step in MTD differs in several respects from that performed in MODE, however. @@ -57,7 +58,7 @@ The most basic change is to use a square convolution filter rather than the circ Another change is that we do not allow any bad data in the convolution square. In MODE, the user may specify what percentage of bad data in the convolution region is permissible, and it will rescale the value of the filter accordingly for each data point. For the sake of speed, MTD requires that there be no bad data in the convolution region. If any bad data exists in the region, the convolved value there is set to a bad data flag. 3D Single Attributes -~~~~~~~~~~~~~~~~~~~~ +-------------------- MTD calculates several 3D attributes for single objects. The object could come from either the forecast field or the observed field. @@ -90,7 +91,7 @@ The **start time** and **end time** of an object are attributes as well. These a Finally, MTD calculates several **intensity percentiles** of the raw data values inside each object. Not all of the attributes are purely geometrical. 3D Pair Attributes -~~~~~~~~~~~~~~~~~~ +------------------ The next category of spatial attributes is for pairs of objects - one of the pair coming from the collection of forecast objects, the other coming from the observation objects. @@ -124,7 +125,7 @@ Finally, the **total interest** gives the result of the fuzzy-logic matching an 2D Constant-Time Attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- The final category of object attributes calculated by MTD are two-dimensional spatial attributes for horizontal (*i.e.*, constant-time) slices of a spacetime object. This is so that the behavior of these attributes over time can be examined. These 2D constant-time attributes are written out for both simple and cluster objects. @@ -141,7 +142,7 @@ For such reasons, having 2D spatial attributes (as in MODE) for each object at e ◦ Axis Angle Matching and Merging -~~~~~~~~~~~~~~~~~~~~ +-------------------- Matching and merging operations in MTD are done in a simpler fashion than in MODE. In order to understand this operation, it is necessary to discuss some very basic notions of graph theory. @@ -177,17 +178,17 @@ To summarize: Any forecast simple objects that find themselves in the same equiv Practical information -_____________________ +===================== MTD input -~~~~~~~~~ +--------- The formats for two-dimensional data files used as input to MTD are the same ones supported by most of the MET tools. Generally speaking, if MODE can use a forecast or observation data file as input, then that file can also be used by MTD. The only difference is that while MODE takes only one forecast and one observed data file as input, MTD takes a series of files. As shown in the next section, filenames for each time used must be given. Thus, for example, if MTD is being used for verification over a period of 24 hours, and the data file valid times are separated by one hour, then a total of 48 filenames must be specified on the MTD command line - 24 filenames for the forecast files, and 24 for the observation files. Further, the filenames must be given in order of increasing valid time. Many users will prefer to write scripts to automate this, rather than type in a lengthy command line by hand. MTD usage -~~~~~~~~~ +--------- The usage statement for the MODE-TD tool is listed below: The command line switches may be given in any order. @@ -237,7 +238,7 @@ An example of the mtd calling sequence is listed below: In this example, the MODE-TD tool will read in a list of forecast GRIB files in the fcst_files directory and similarly spaced observation GRIB files in the obs_files directory. It uses a configuration file called **MTDConfig_default** and writes the output to the *out_dir/mtd* directory. MTD configuration file -~~~~~~~~~~~~~~~~~~~~~~ +---------------------- The default configuration file for the MODE tool, **MODEConfig_default**, can be found in the installed *share/met/config* directory. Another version of the configuration file is provided in *scripts/config*. We encourage users to make a copy of the configuration files prior to modifying their contents.Most of the entries in the MTD configuration file should be familiar from the corresponding file for MODE. This initial beta release of MTD does not offer all the tunable options that MODE has accumulated over the years, however. In this section, we will not bother to repeat explanations of config file details that are exactly the same as those in MODE; we will only explain those elements that are different from MODE, and those that are unique to MTD. @@ -351,7 +352,7 @@ ______________________ The **txt_output** dictionary also contains a collection of boolean flags, in this case controlling the output of ASCII attribute files. The **attributes_2d** flag controls the output of the 2D object attributes for constant-time slices of 3D objects, while the **attributes_3d** flag controls the output of single and pair 3D spacetime object attributes. mtd output -~~~~~~~~~~ +---------- MTD creates several output files after each run in ASCII and NetCDF formats. There are text files giving 2D and 3D attributes of spacetime objects and information on matches and merges, as well as a NetCDF file giving the objects themselves, in case any further or specialized analysis of the objects needs to be done. diff --git a/met/docs/Users_Guide/mode.rst b/met/docs/Users_Guide/mode.rst index ab341e7cce..a0df6932da 100644 --- a/met/docs/Users_Guide/mode.rst +++ b/met/docs/Users_Guide/mode.rst @@ -1,12 +1,13 @@ .. _mode: +********* MODE Tool -========= +********* .. _MODE_Introduction: Introduction -____________ +============ This section provides a description of the Method for Object-Based Diagnostic Evaluation (MODE) tool, which was developed at the Research Applications Laboratory, NCAR/Boulder, USA. More information about MODE can be found in :ref:`Davis et al. (2006a,b) `, :ref:`Brown et al. (2007) ` and :ref:`Bullock et al. (2016) `. @@ -17,12 +18,12 @@ MODE may be used in a generalized way to compare any two fields. For simplicity, .. _MODE_Scientific-and-statistical: Scientific and statistical aspects -__________________________________ +================================== The methods used by the MODE tool to identify and match forecast and observed objects are briefly described in this section. Resolving objects -~~~~~~~~~~~~~~~~~ +----------------- The process used for resolving objects in a raw data field is called *convolution thresholding*. The raw data field is first convolved with a simple filter function as follows: @@ -58,7 +59,7 @@ An example of the steps involved in resolving objects is shown in :numref:`mode- Attributes -~~~~~~~~~~ +---------- Object attributes are defined both for single objects and for object pairs. One of the objects in a pair is from the forecast field and the other is taken from the observed field. @@ -81,7 +82,7 @@ All the attributes discussed so far are defined for single objects. Once these a Several area measures are also used for pair attributes. **Union Area** is the total area that is in either one (or both) of the two objects. **Intersection Area** is the area that is inside both objects simultaneously. **Symmetric Difference** is the area inside at least one object, but not inside both. Fuzzy logic -~~~~~~~~~~~ +----------- Once object attributes :math:`\alpha_1,\alpha_2,\ldots,\alpha_n` are estimated, some of them are used as input to a fuzzy logic engine that performs the matching and merging steps. **Merging** refers to grouping together objects in a single field, while **matching** refers to grouping together objects in different fields, typically the forecast and observed fields. Interest maps :math:`I_i` are applied to the individual attributes :math:`\alpha_i` to convert them into interest values, which range from zero (representing no interest) to one (high interest). For example, the default interest map for centroid difference is one for small distances, and falls to zero as the distance increases. For other attributes (*e.g.*, intersection area), low values indicate low interest, and high values indicate more interest. @@ -98,19 +99,19 @@ This total interest value is then thresholded, and pairs of objects that have to Another merging method is available in MODE, which can be used instead of, or along with, the fuzzy logic based merging just described. Recall that the convolved field is thresholded to produce the mask field. A second (lower) threshold can be specified so that objects that are separated at the higher threshold but joined at the lower threshold are merged. Summary statistics -~~~~~~~~~~~~~~~~~~ +------------------ Once MODE has been run, summary statistics are written to an output file. These files contain information about all single and cluster objects and their attributes. Total interest for object pairs is also output, as are percentiles of intensity inside the objects. The output file is in a simple flat ASCII tabular format (with one header line) and thus should be easily readable by just about any programming language, scripting language, or statistics package. Refer to :numref:`MODE-output` for lists of the statistics included in the MODE output files. Example scripts will be posted on the MET website in the future. Practical information -_____________________ +===================== This section contains a description of how MODE can be configured and run. The MODE tool is used to perform a features-based verification of gridded model data using gridded observations. The input gridded model and observation datasets must be in one of the MET supported gridded file formats. The requirement of having all gridded fields using the same grid specification has been removed with METv5.1. The Grid-Stat tool performs no interpolation when the input model, observation, and climatology datasets must be on a common grid. MET will interpolate these files to a common grid if one is specified. There is a regrid option in the configuration file that allows the user to define the grid upon which the scores will be computed. The gridded analysis data may be based on observations, such as Stage II or Stage IV data for verifying accumulated precipitation, or a model analysis field may be used. However, users are cautioned that it is generally unwise to verify model output using an analysis field produced by the same model. MODE provides the capability to select a single model variable/level from which to derive objects to be analyzed. MODE was developed and tested using accumulated precipitation. However, the code has been generalized to allow the use of any gridded model and observation field. Based on the options specified in the configuration file, MODE will define a set of simple objects in the model and observation fields. It will then compute an interest value for each pair of objects across the fields using a fuzzy engine approach. Those interest values are thresholded, and any pairs of objects above the threshold will be matched/merged. Through the configuration file, MODE offers a wide range of flexibility in how the objects are defined, processed, matched, and merged. mode usage -~~~~~~~~~~ +---------- The usage statement for the MODE tool is listed below: @@ -177,7 +178,7 @@ In Example 2, the MODE tool will verify the model data in the sample_fcst.nc Net .. _MODE-configuration-file: mode configuration file -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- The default configuration file for the MODE tool, **MODEConfig_default**, can be found in the installed *share/met/config* directory. Another version of the configuration file is provided in *scripts/config*. We encourage users to make a copy of the configuration files prior to modifying their contents. Descriptions of **MODEConfig_default** and the required variables for any MODE configuration file are also provided below. While the configuration file contains many entries, most users will only need to change a few for their use. Specific options are described in the following subsections. @@ -493,7 +494,7 @@ When MODE is run on global grids, this parameter specifies how many grid squares .. _MODE-output: mode output -~~~~~~~~~~~ +----------- MODE produces output in ASCII, NetCDF, and PostScript formats. diff --git a/met/docs/Users_Guide/overview.rst b/met/docs/Users_Guide/overview.rst index ac1d55c067..8eed622ede 100644 --- a/met/docs/Users_Guide/overview.rst +++ b/met/docs/Users_Guide/overview.rst @@ -1,10 +1,11 @@ .. _overview: +*************** Overview of MET -=============== +*************** Purpose and organization of the User's Guide -____________________________________________ +============================================ The goal of this User's Guide is to provide basic information for users of the Model Evaluation Tools (MET) to enable them to apply MET to their datasets and evaluation studies. MET was originally designed for application to the post-processed output of the `Weather Research and Forecasting (WRF) `_ model. However, MET may also be used for the evaluation of forecasts from other models or applications, including the `Unified Forecast System (UFS) `_, and the `System for Integrated Modeling of the Atmosphere (SIMA) `_ if certain file format definitions (described in this document) are followed. @@ -13,7 +14,7 @@ The MET User's Guide is organized as follows. :numref:`overview` provides an ove The remainder of this section includes information about the context for MET development, as well as information on the design principles used in developing MET. In addition, this section includes an overview of the MET package and its specific modules. The Developmental Testbed Center (DTC) -______________________________________ +====================================== MET has been developed, and will be maintained and enhanced, by the `Developmental Testbed Center (DTC) `_. The main goal of the DTC is to serve as a bridge between operations and research, to facilitate the activities of these two important components of the numerical weather prediction (NWP) community. The DTC provides an environment that is functionally equivalent to the operational environment in which the research community can test model enhancements; the operational community benefits from DTC testing and evaluation of models before new models are implemented operationally. MET serves both the research and operational communities in this way - offering capabilities for researchers to test their own enhancements to models and providing a capability for the DTC to evaluate the strengths and weaknesses of advances in NWP prior to operational implementation. @@ -22,7 +23,7 @@ The MET package is available to DTC staff, visitors, and collaborators, as well .. _MET-goals: MET goals and design philosophy -_______________________________ +=============================== The primary goal of MET development is to provide a state-of-the-art verification package to the NWP community. By "state-of-the-art" we mean that MET will incorporate newly developed and advanced verification methodologies, including new methods for diagnostic and spatial verification and new techniques provided by the verification and modeling communities. MET also utilizes and replicates the capabilities of existing systems for verification of NWP forecasts. For example, the MET package replicates existing National Center for Environmental Prediction (NCEP) operational verification capabilities (e.g., I/O, methods, statistics, data types). MET development will take into account the needs of the NWP community - including operational centers and the research and development community. Some of the MET capabilities include traditional verification approaches for standard surface and upper air variables (e.g., Equitable Threat Score, Mean Squared Error), confidence intervals for verification measures, and spatial forecast verification methods. In the future, MET will include additional state-of-the-art and new methodologies. @@ -31,7 +32,7 @@ The MET package has been designed to be modular and adaptable. For example, indi The MET code and documentation is maintained by the DTC in Boulder, Colorado. The MET package is freely available to the modeling, verification, and operational communities, including universities, governments, the private sector, and operational modeling and prediction centers. MET components -______________ +============== The major components of the MET package are represented in :numref:`overview-figure`. The main stages represented are input, reformatting, plotting, intermediate output, statistical analyses, and output and aggregation/analysis. The MET-TC package functions independently of the other MET modules, as indicated in the Figure. Each of these stages is described further in later sections. For example, the input and output formats are discussed in :numref:`data_io` as well as in the sections associated with each of the statistics modules. MET input files are represented on the far left. @@ -74,12 +75,12 @@ The following sections of this MET User's Guide contain usage statements for eac .. include:: release-notes.rst Future development plans -________________________ +======================== MET is an evolving verification software package. New capabilities are planned in controlled, successive version releases. Bug fixes and user-identified problems will be addressed as they are found and posted to the known issues section of the `MET User Support web page `_. Plans are also in place to incorporate many new capabilities and options in future releases of MET. Please refer to the issues listed in the `MET GitHub repository `_ to see our development priorities for upcoming releases. Code support -____________ +============ MET support is provided through the `METplus GitHub Discussions Forum `_. We will endeavor to respond to requests for help in a timely fashion. In addition, information about MET and tools that can be used with MET are provided on the `MET web page `_. @@ -90,7 +91,7 @@ We welcome comments and suggestions for improvements to MET, especially informat The MET package is a "living" set of tools. Our goal is to continually enhance it and add to its capabilities. Because our time, resources, and talents are limited, we welcome contributed code for future versions of MET. These contributions may represent new verification methodologies, new analysis tools, or new plotting functions. For more information on contributing code to MET, please create a post in the `METplus GitHub Discussions Forum `_. Fortify -_______ +======= Requirements from various government agencies that use MET have resulted in our code being analyzed by Fortify, a proprietary static source code analyzer owned by HP Enterprise Security Products. Fortify analyzes source code to identify for security risks, memory leaks, uninitialized variables, and other such weaknesses and bad coding practices. Fortify categorizes any issues it finds as low priority, high priority, or critical, and reports these issues back to the developers for them to address. A development cycle is thus established, with Fortify analyzing code and reporting back to the developers, who then make changes in the source code to address these issues, and hand the new code off to Fortify again for re-analysis. The goal is to drive the counts of both high priority and critical issues down to zero. diff --git a/met/docs/Users_Guide/plotting.rst b/met/docs/Users_Guide/plotting.rst index a96dca605a..9b08a2be7e 100644 --- a/met/docs/Users_Guide/plotting.rst +++ b/met/docs/Users_Guide/plotting.rst @@ -1,10 +1,11 @@ .. _plotting: +***************************** Plotting and Graphics Support -============================= +***************************** Plotting Utilities -__________________ +================== This section describes how to check your data files using plotting utilities. Point observations can be plotted using the Plot-Point-Obs utility. A single model level can be plotted using the plot_data_plane utility. For object based evaluations, the MODE objects can be plotted using plot_mode_field. Occasionally, a post-processing or timing error can lead to errors in MET. These tools can assist the user by showing the data to be verified to ensure that times and locations match up as expected. @@ -12,7 +13,7 @@ MET version 10.1 adds support for the passing point observations to plot_point_o plot_point_obs usage -~~~~~~~~~~~~~~~~~~~~ +-------------------- The usage statement for the Plot-Point-Obs utility is shown below: @@ -82,7 +83,7 @@ Please refer to :numref:`Appendix F, Section %s ` for more details ab plot_point_obs configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------- The default configuration file for the Plot-Point-Obs tool named **PlotPointObsConfig_default** can be found in the installed *share/met/config* directory. The contents of the configuration file are described in the subsections below. @@ -219,7 +220,7 @@ For each observation, this tool stores the observation latitude, longitude, and .. _plot_data_plane-usage: plot_data_plane usage -~~~~~~~~~~~~~~~~~~~~~ +--------------------- The usage statement for the plot_data_plane utility is shown below: @@ -274,7 +275,7 @@ A second example of the plot_data_plane calling sequence is shown below: In the first example, the Plot-Data-Plane tool will process the input test.grb file and write a PostScript image to a file named test.ps showing temperature at 2 meters. The second example plots downward shortwave radiation flux at the surface. The second example is run at verbosity level 4 so that the user can inspect the output and make sure its plotting the intended record. plot_mode_field usage -~~~~~~~~~~~~~~~~~~~~~ +--------------------- The usage statement for the plot_mode_field utility is shown below: @@ -326,10 +327,10 @@ In this example, the plot_mode_field tool will plot simple objects from an obser Once MET has been applied to forecast and observed fields (or observing locations), and the output has been sorted through the Analysis Tool, numerous graphical and summary analyses can be performed depending on a specific user's needs. Here we give some examples of graphics and summary scores that one might wish to compute with the given output of MET and MET-TC. Any computing language could be used for this stage; some scripts will be provided on the `MET users web page `_ as examples to assist users. Examples of plotting MET output -_______________________________ +=============================== Grid-Stat tool examples -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- The plots in :numref:`plotting_Gilbert_skill_score` show time series of frequency bias and Gilbert Skill Score, stratified according to time of day. This type of figure is particularly useful for diagnosing problems that are tied to the diurnal cycle. In this case, two of the models (green dash-dotted and black dotted lines) show an especially high Bias (near 3) during the afternoon (15-21 UTC; left panel), while the skill (GSS; right panel) appears to be best for the models represented by the solid black line and green dashed lines in the morning (09-15 UTC). Note that any judgment of skill based on GSS should be restricted to times when the Bias is close to one. @@ -340,7 +341,7 @@ The plots in :numref:`plotting_Gilbert_skill_score` show time series of frequenc Time series of forecast area bias and Gilbert Skill Score for four model configurations (different lines) stratified by time-of-day. MODE tool examples -~~~~~~~~~~~~~~~~~~ +------------------ When using the MODE tool, it is possible to think of matched objects as hits and unmatched objects as false alarms or misses depending on whether the unmatched object is from the forecast or observed field, respectively. Because the objects can have greatly differing sizes, it is useful to weight the statistics by the areas, which are given in the output as numbers of grid squares. When doing this, it is possible to have different matched observed object areas from matched forecast object areas so that the number of hits will be different depending on which is chosen to be a hit. When comparing multiple forecasts to the same observed field, it is perhaps wise to always use the observed field for the hits so that there is consistency for subsequent comparisons. Defining hits, misses and false alarms in this way allows one to compute many traditional verification scores without the problem of small-scale discrepancies; the matched objects are defined as being matched because they are "close" by the fuzzy logic criteria. Note that scores involving the number of correct negatives may be more difficult to interpret as it is not clear how to define a correct negative in this context. It is also important to evaluate the number and area attributes for these objects in order to provide a more complete picture of how the forecast is performing. @@ -363,7 +364,7 @@ In addition to the traditional scores, MODE output allows more information to be .. _TC-Stat-tool-example: TC-Stat tool example -~~~~~~~~~~~~~~~~~~~~ +-------------------- There is a basic R script located in the MET installation, *share/met/Rscripts/plot_tcmpr.R*. The usage statement with a short description of the options for *plot_tcmpr.R* can be obtained by typing: Rscript *plot_tcmpr.R* with no additional arguments. The only required argument is the **-lookin** source, which is the path to the TC-Pairs TCST output files. The R script reads directly from the TC-Pairs output, and calls TC-Stat directly for filter jobs specified in the *"-filter options"* argument. diff --git a/met/docs/Users_Guide/point-stat.rst b/met/docs/Users_Guide/point-stat.rst index ce7b74e905..5c89be5d6d 100644 --- a/met/docs/Users_Guide/point-stat.rst +++ b/met/docs/Users_Guide/point-stat.rst @@ -1,24 +1,25 @@ .. _point-stat: +*************** Point-Stat Tool -=============== +*************** Introduction -____________ +============ The Point-Stat tool provides verification statistics for forecasts at observation points (as opposed to over gridded analyses). The Point-Stat tool matches gridded forecasts to point observation locations and supports several different interpolation options. The tool then computes continuous, categorical, spatial, and probabilistic verification statistics. The categorical and probabilistic statistics generally are derived by applying a threshold to the forecast and observation values. Confidence intervals - representing the uncertainty in the verification measures - are computed for the verification statistics. Scientific and statistical aspects of the Point-Stat tool are discussed in the following section. Practical aspects of the Point-Stat tool are described in :numref:`tc-stat_practical-information`. Scientific and statistical aspects -__________________________________ +================================== The statistical methods and measures computed by the Point-Stat tool are described briefly in this section. In addition, :numref:`matching-methods` discusses the various interpolation options available for matching the forecast grid point values to the observation points. The statistical measures computed by the Point-Stat tool are described briefly in :numref:`PS_Statistical-measures` and in more detail in :numref:`Appendix C, Section %s `. :numref:`PS_Statistical-confidence-intervals` describes the methods for computing confidence intervals that are applied to some of the measures computed by the Point-Stat tool; more detail on confidence intervals is provided in :numref:`Appendix D, Section %s `. .. _matching-methods: Interpolation/matching methods -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ This section provides information about the various methods available in MET to match gridded model output to point observations. Matching in the vertical and horizontal are completed separately using different methods. @@ -119,7 +120,7 @@ The forecast value at P is chosen as the grid point inside the interpolation are .. _PS_HiRA_framework: HiRA framework -~~~~~~~~~~~~~~ +-------------- The Point-Stat tool has been enhanced to include the High Resolution Assessment (HiRA) verification logic (:ref:`Mittermaier, 2014 `). HiRA is analogous to neighborhood verification but for point observations. The HiRA logic interprets the forecast values surrounding each point observation as an ensemble forecast. These ensemble values are processed in three ways. First, the ensemble continuous statistics (ECNT), the observation rank statistics (ORANK) and the ranked probability score (RPS) line types are computed directly from the ensemble values. Second, for each categorical threshold specified, a fractional coverage value is computed as the ratio of the nearby forecast values that meet the threshold criteria. Point-Stat evaluates those fractional coverage values as if they were a probability forecast. When applying HiRA, users should enable the matched pair (MPR), probabilistic (PCT, PSTD, PJC, or PRC), continuous ensemble statistics (ECNT), observation rank statistics (ORANK) or ranked probability score (RPS) line types in the **output_flag** dictionary. The number of probabilistic HiRA output lines is determined by the number of categorical forecast thresholds and HiRA neighborhood widths chosen. @@ -138,7 +139,7 @@ Often, the neighborhood size is chosen so that multiple models to be compared ha .. _PS_Statistical-measures: Statistical measures -~~~~~~~~~~~~~~~~~~~~ +-------------------- The Point-Stat tool computes a wide variety of verification statistics. Broadly speaking, these statistics can be subdivided into statistics for categorical variables and statistics for continuous variables. The categories of measures are briefly described here; specific descriptions of the measures are provided in :numref:`Appendix C, Section %s `. Additional information can be found in :ref:`Wilks (2011) ` and :ref:`Jolliffe and Stephenson (2012) `, and at Collaboration for Australian Weather and Climate Research. Forecast Verification - `Issues, Methods and FAQ web page. `_ @@ -177,7 +178,7 @@ Often, the sample climatology is used as a reference by a skill score. The sampl .. _PS_Statistical-confidence-intervals: Statistical confidence intervals -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------- A single summary score gives an indication of the forecast performance, but it is a single realization from a random process that neglects uncertainty in the score's estimate. That is, it is possible to obtain a good score, but it may be that the "good" score was achieved by chance and does not reflect the "true" score. Therefore, when interpreting results from a verification analysis, it is imperative to analyze the uncertainty in the realized scores. One good way to do this is to utilize confidence intervals. A confidence interval indicates that if the process were repeated many times, say 100, then the true score would fall within the interval :math:`100(1-\alpha)\%` of the time. Typical values of :math:`\alpha` are 0.01, 0.05, and 0.10. The Point-Stat tool allows the user to select one or more specific :math:`\alpha`-values to use. @@ -248,14 +249,14 @@ For more information on confidence intervals pertaining to verification measures .. _tc-stat_practical-information: Practical information -_____________________ +===================== The Point-Stat tool is used to perform verification of a gridded model field using point observations. The gridded model field to be verified must be in one of the supported file formats. The point observations must be formatted as the NetCDF output of the point reformatting tools described in :numref:`reformat_point`. The Point-Stat tool provides the capability of interpolating the gridded forecast data to the observation points using a variety of methods as described in :numref:`matching-methods`. The Point-Stat tool computes a number of continuous statistics on the matched pair data as well as discrete statistics once the matched pair data have been thresholded. If no matched pairs are found for a particular verification task, a report listing counts for reasons why the observations were not used is written to the log output at the default verbosity level of 2. If matched pairs are found, this report is written at verbosity level 3. Inspecting these rejection reason counts is the first step in determining why Point-Stat found no matched pairs. The order of the log messages matches the order in which the processing logic is applied. Start from the last log message and work your way up, considering each of the non-zero rejection reason counts. point_stat usage -~~~~~~~~~~~~~~~~ +---------------- The usage statement for the Point-Stat tool is shown below: @@ -309,7 +310,7 @@ An example of the point_stat calling sequence is shown below: In this example, the Point-Stat tool evaluates the model data in the sample_fcst.grb GRIB file using the observations in the NetCDF output of PB2NC, sample_pb.nc, applying the configuration options specified in the **PointStatConfig file**. point_stat configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------- The default configuration file for the Point-Stat tool named **PointStatConfig_default** can be found in the installed *share/met/config* directory. Another version is located in *scripts/config*. We encourage users to make a copy of these files prior to modifying their contents. The contents of the configuration file are described in the subsections below. @@ -483,7 +484,7 @@ If all line types corresponding to a particular verification method are set to N .. _point_stat-output: point_stat output -~~~~~~~~~~~~~~~~~ +----------------- point_stat produces output in STAT and, optionally, ASCII format. The ASCII output duplicates the STAT output but has the data organized by line type. The output files will be written to the default output directory or the directory specified using the "-outdir" command line option. diff --git a/met/docs/Users_Guide/reformat_grid.rst b/met/docs/Users_Guide/reformat_grid.rst index 3f67a377a7..c3a1e438ee 100644 --- a/met/docs/Users_Guide/reformat_grid.rst +++ b/met/docs/Users_Guide/reformat_grid.rst @@ -1,12 +1,13 @@ .. _reformat_grid: +******************************* Re-Formatting of Gridded Fields -=============================== +******************************* Several MET tools exist for the purpose of reformatting gridded fields, and they are described in this section. These tools are represented by the reformatting column of MET flowchart depicted in :numref:`overview-figure`. Pcp-Combine tool -________________ +================ This section describes the Pcp-Combine tool which summarizes data across multiple input gridded data files and writes the results to a single NetCDF output file. It is often used to modify precipitation accumulation intervals in the forecast and/or observation datasets to make them comparable. However it can also be used to derive summary fields, such as daily min/max temperature or average precipitation rate. @@ -23,7 +24,7 @@ The Pcp-Combine tool supports four types of commands ("sum", "add", "subtract", By default, the Pcp-Combine tool processes data for **APCP**, the GRIB string for accumulated precipitation. When requesting data using time strings (i.e. [HH]MMSS), Pcp-Combine searches for accumulated precipitation for that accumulation interval. Alternatively, use the "-field" option to process fields other than **APCP** or for non-GRIB files. The "-field" option may be used multiple times to process multiple fields in a single run. Since the Pcp-Combine tool does not support automated regridding, all input data must be on the same grid. In general the input files should have the same initialization time unless the user has indicated that it should ignore the initialization time for the "sum" command. The "subtract" command produces a warning when the input initialization times differ or the subtraction results in a negative accumulation interval. pcp_combine usage -~~~~~~~~~~~~~~~~~ +----------------- The usage statement for the Pcp-Combine tool is shown below: @@ -161,7 +162,7 @@ The Pcp-Combine tool will search for 24 files containing 1-hourly accumulation i This command would grab the first level of the TT variable from a pinterp NetCDF file and write it to the output tt_10.nc file. pcp_combine output -~~~~~~~~~~~~~~~~~~ +------------------ The output NetCDF files contain the requested accumulation intervals as well as information about the grid on which the data lie. That grid projection information will be parsed out and used by the MET statistics tools in subsequent steps. One may use NetCDF utilities such as ncdump or ncview to view the contents of the output file. Alternatively, the MET Plot-Data-Plane tool described in :numref:`plot_data_plane-usage` may be run to create a PostScript image of the data. @@ -204,12 +205,12 @@ Each NetCDF file generated by the Pcp-Combine tool contains the dimensions and v .. _regrid-data-plane: Regrid-Data-Plane tool -______________________ +====================== This section contains a description of running the Regrid-Data-Plane tool. This tool may be run to read data from any gridded file MET supports, interpolate to a user-specified grid, and writes the field(s) out in NetCDF format. The user may specify the method of interpolation used for regridding as well as which fields to regrid. This tool is particularly useful when dealing with GRIB2 and NetCDF input files that need to be regridded. For GRIB1 files, it has also been tested for compatibility with the copygb regridding utility mentioned in :numref:`Installation-of-optional`. regrid_data_plane usage -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- The usage statement for the regrid_data_plane utility is shown below: @@ -283,17 +284,17 @@ For more details on setting the **to_grid, -method, -width,** and **-vld_thresh* In this example, the Regrid-Data-Plane tool will regrid data from the **input.grb** file to the grid on which the first record of the **togrid.grb** file resides using Bilinear Interpolation with a width of 2 and write the output in NetCDF format to a file named **regridded.nc**. The variables in **regridded.nc** will include 6-hour accumulated precipitation, 2m temperature, 10m U and V components of the wind, and the 500mb geopotential height. Automated regridding within tools -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------- While the Regrid-Data-Plane tool is useful as a stand-alone tool, the capability is also included to automatically **regrid** one or both fields in most of the MET tools that handle gridded data. See the regrid entry in :numref:`Configuration File Details` for a description of the configuration file entries that control automated regridding. Shift-Data-Plane tool -_____________________ +===================== The Shift-Data-Plane tool performs a rigid shift of the entire grid based on user-defined specifications and writes the field(s) out in NetCDF format. This tool was originally designed to account for track error when comparing fields associated with tropical cyclones. The user specifies the latitude and longitude of the source and destination points to define the shift. Both points must fall within the domain and are used to define the X and Y direction grid unit shift. The shift is then applied to all grid points. The user may specify the method of interpolation and the field to be shifted. The effects of topography and land/water masks are ignored. shift_data_plane usage -~~~~~~~~~~~~~~~~~~~~~~ +---------------------- The usage statement for the shift_data_plane utility is shown below: @@ -357,12 +358,12 @@ For more details on setting the **-method** and **-width** options, see the **re In this example, the Shift-Data-Plane tool reads 12-hour accumulated precipitation from the **nam.grb** file, applies a rigid shift defined by (38.6272, -90.1978) to (40.1717, -105.1092) and writes the output in NetCDF format to a file named **nam_shift_APCP_12.nc**. These **-from** and **-to** locations result in a grid shift of -108.30 units in the x-direction and 16.67 units in the y-direction. MODIS regrid tool -_________________ +================= This section contains a description of running the MODIS regrid tool. This tool may be run to create a NetCDF file for use in other MET tools from `MODIS level 2 cloud product from NASA. `_ modis_regrid usage -~~~~~~~~~~~~~~~~~~ +------------------ The usage statement for the modis_regrid utility is shown below: @@ -427,14 +428,14 @@ In this example, the Modis-Regrid tool will process the Cloud_Fraction field fro Example plot showing surface temperature from a MODIS file. WWMCA Tool Documentation -________________________ +======================== There are two WWMCA tools available. The WWMCA-Plot tool makes a PostScript plot of one or more WWMCA cloud percent files and the WWMCA-Regrid tool regrids binary WWMCA data files and reformats them into NetCDF files that the other MET tools can read. The WWMCA-Regrid tool has been generalized to more broadly support any data stored in the WWMCA binary format. The WWMCA tools attempt to parse timing and hemisphere information from the file names. They tokenize the filename using underscores (_) and dots (.) and examine each element which need be in no particular order. A string of 10 or more numbers is interpreted as the valid time in YYYYMMDDHH[MMSS] format. The string NH indicates the northern hemisphere while SH indicates the southern hemisphere. While WWMCA data is an analysis and has no forecast lead time, other datasets following this format may. Therefore, a string of 1 to 4 numbers is interpreted as the forecast lead time in hours. While parsing the filename provides default values for this timing information, they can be overridden by explicitly setting their values in the WWMCA-Regrid configuration file. wwmca_plot usage -~~~~~~~~~~~~~~~~ +---------------- The usage statement for the WWMCA-Plot tool is shown below: @@ -470,7 +471,7 @@ Optional arguments for wwmca_plot Example output of WWMCA-Plot tool. wwmca_regrid usage -~~~~~~~~~~~~~~~~~~ +------------------ The usage statement for the WWMCA-Regrid tool is shown below: @@ -510,7 +511,7 @@ Optional arguments for wwmca_regrid In any regridding problem, there are two grids involved: the "From" grid, which is the grid the input data are on, and the "To" grid, which is the grid the data are to be moved onto. In **WWMCA-Regrid** the "From" grid is pre-defined by the hemisphere of the WWMCA binary files being processed. The "To" grid and corresponding regridding logic are specified using the **regrid** section of the configuration file. If the "To" grid is entirely confined to one hemisphere, then only the WWMCA data file for that hemisphere needs to be given. If the "To" grid or the interpolation box used straddles the equator, the data files for both hemispheres need to be given. Once the "To" grid is specified in the config file, the WWMCA-Regrid tool will know which input data files it needs and will complain if it is not given the right ones. wwmca_regrid configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- The default configuration file for the WWMCA-Regrid tool named **WWMCARegridConfig_default** can be found in the installed *share/met/config* directory. We encourage users to make a copy of this file prior to modifying its contents. The contents of the configuration file are described in the subsections below. diff --git a/met/docs/Users_Guide/reformat_point.rst b/met/docs/Users_Guide/reformat_point.rst index 9e41ca80dc..7a3fce768f 100644 --- a/met/docs/Users_Guide/reformat_point.rst +++ b/met/docs/Users_Guide/reformat_point.rst @@ -1,21 +1,22 @@ .. _reformat_point: +*********************************** Re-Formatting of Point Observations -=================================== +*********************************** There are several formats of point observations that may be preprocessed using the suite of reformatting tools in MET. These include PrepBUFR data from NCEP, SURFRAD data from NOAA, AERONET data from NASA, MADIS data from NOAA, little_r from WRF simulations, and user-defined data in a generic ASCII format. These steps are represented by the first columns in the MET flowchart depicted in :numref:`overview`. The software tools used to reformat point data are described in this section. .. _PB2NC tool: PB2NC tool -__________ +========== This section describes how to configure and run the PB2NC tool. The PB2NC tool is used to stratify the contents of an input PrepBUFR point observation file and reformat it into NetCDF format for use by other MET tools. The PB2NC tool must be run on the input PrepBUFR point observation file prior to performing verification with the MET statistics tools. .. _pb2nc usage: pb2nc usage -~~~~~~~~~~~ +----------- The usage statement for the PB2NC tool is shown below: @@ -91,7 +92,7 @@ In this example, the PB2NC tool will process the input **sample_pb.blk** file ap .. _pb2nc configuration file: pb2nc configuration file -~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------ The default configuration file for the PB2NC tool named **PB2NCConfig_default** can be found in the installed *share/met/config* directory. The version used for the example run in :numref:`Sample test cases` is available in *scripts/config*. It is recommended that users make a copy of configuration files prior to modifying their contents. @@ -103,7 +104,7 @@ For example, using an environment variable to set the **message_type** (see belo \* In the configuration file: **message_type = [ ${MSG_TYP} ];** - The contents of the default pb2nc configuration file are described below. +The contents of the default pb2nc configuration file are described below. ____________________ @@ -124,7 +125,7 @@ _____________________ Each PrepBUFR message is tagged with one of eighteen message types as listed in the :numref:`config_options` file. The **message_type** refers to the type of observation from which the observation value (or 'report') was derived. The user may specify a comma-separated list of message types to be retained. Providing an empty list indicates that all message types should be retained. -___________________ +_____________________ .. code-block:: none @@ -132,7 +133,7 @@ ___________________ The **message_type_map** entry is an array of dictionaries, each containing a **key** string and **val** string. This defines a mapping of input PrepBUFR message types to output message types. This provides a method for renaming input PrepBUFR message types. -______________________ +_____________________ .. code-block:: none @@ -146,7 +147,7 @@ ______________________ The **message_type_group_map** entry is an array of dictionaries, each containing a **key** string and **val** string. This defines a mapping of message type group names to a comma-separated list of values. This map is defined in the config files for PB2NC, Point-Stat, or Ensemble-Stat. Modify this map to define sets of message types that should be processed together as a group. The **SURFACE** entry must be present to define message types for which surface verification logic should be applied. -______________ +_____________________ .. code-block:: none @@ -154,7 +155,7 @@ ______________ Each PrepBUFR message has a station identification string associated with it. The user may specify a comma-separated list of station IDs to be retained. Providing an empty list indicates that messages from all station IDs will be retained. It can be a file name containing a list of stations. -_______________ +_____________________ .. code-block:: none @@ -163,7 +164,7 @@ _______________ The **beg** and **end** variables are used to stratify the elevation (in meters) of the observations to be retained. The range shown above is set to -1000 to 100000 meters, which essentially retains every observation. -__________________ +_____________________ .. code-block:: none @@ -178,7 +179,7 @@ The **pb_report_type, in_report_type**, and **instrument_type** variables are us `PrepBUFR Code table for input report types. `_ -_________________ +_____________________ .. code-block:: none @@ -219,8 +220,8 @@ The **level_category** variable is used to specify a comma-separated list of Pre * - 7 - Auxiliary levels generated via interpolation from spanning levels -_______________ - +_____________________ + .. code-block:: none obs_bufr_var = [ 'QOB', 'TOB', 'ZOB', 'UOB', 'VOB' ]; @@ -228,7 +229,7 @@ _______________ Each PrepBUFR message will likely contain multiple observation variables. The **obs_bufr_var** variable is used to specify which observation variables should be retained or derived. The variable name comes from BUFR file which includes BUFR table. The following BUFR names may be retained: QOB, TOB, ZOB, UOB, and VOB for specific humidity, temperature, height, and the u and v components of winds. The following BUFR names may be derived: D_DPT, D_WIND, D_RH, D_MIXR, D_PRMSL, D_PBL, and D_CAPE for dew point, wind speed, relative humidity, mixing ratio, pressure reduced to MSL, planetary boundary layer height, and convective available potential energy. This configuration replaces **obs_grib_code**. If the list is empty, all BUFR variables are retained. -________________ +_____________________ .. code-block:: none @@ -252,7 +253,7 @@ ________________ The BUFR variable names are not shared with other forecast data. This map is used to convert the BUFR name to the common name, like GRIB2. It allows to share the configuration for forecast data with PB2NC observation data. If there is no mapping, the BUFR variable name will be saved to output NetCDF file. -______________ +_____________________ .. code-block:: none @@ -261,7 +262,7 @@ ______________ Each observation has a quality mark value associated with it. The **quality_mark_thresh** is used to stratify out which quality marks will be retained. The value shown above indicates that only observations with quality marks less than or equal to 2 will be retained. -_________________ +_____________________ .. code-block:: none @@ -270,7 +271,7 @@ _________________ A PrepBUFR message may contain duplicate observations with different quality mark values. The **event_stack_flag** indicates whether to use the observations at the top of the event stack (observation values have had more quality control processing applied) or the bottom of the event stack (observation values have had no quality control processing applied). The flag value of **TOP** listed above indicates the observations with the most amount of quality control processing should be used, the **BOTTOM** option uses the data closest to raw values. -___________________ +_____________________ .. code-block:: none @@ -311,7 +312,7 @@ The **vld_freq** and **vld_thresh** entries specify the required ratio of valid .. _pb2nc output: pb2nc output -~~~~~~~~~~~~ +------------ Each NetCDF file generated by the PB2NC tool contains the dimensions and variables shown in :numref:`table_reform-point_pb2nc_output_dim` and :numref:`table_reform-point_pb2nc_output_vars`. @@ -416,7 +417,7 @@ Each NetCDF file generated by the PB2NC tool contains the dimensions and variabl ASCII2NC tool -_____________ +============= This section describes how to run the ASCII2NC tool. The ASCII2NC tool is used to reformat ASCII point observations into the NetCDF format expected by the Point-Stat tool. For those users wishing to verify against point observations that are not available in PrepBUFR format, the ASCII2NC tool provides a way of incorporating those observations into MET. If the ASCII2NC tool is used to perform a reformatting step, no configuration file is needed. However, for more complex processing, such as summarizing time series observations, a configuration file may be specified. For details on the configuration file options, see :numref:`config_options` and example configuration files distributed with the MET code. @@ -473,7 +474,7 @@ The default ASCII point observation format consists of one row of data per obser - Observation value in units consistent with the GRIB code definition. ascii2nc usage -~~~~~~~~~~~~~~ +-------------- Once the ASCII point observations have been formatted as expected, the ASCII file is ready to be processed by the ASCII2NC tool. The usage statement for ASCII2NC tool is shown below: @@ -544,13 +545,13 @@ Here is an example of processing the same set of observations but using Python e Please refer to :numref:`Appendix F, Section %s ` for more details about Python embedding in MET. ascii2nc configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- The default configuration file for the ASCII2NC tool named **Ascii2NcConfig_default** can be found in the installed *share/met/config* directory. It is recommended that users make a copy of this file prior to modifying its contents. The ASCII2NC configuration file is optional and only necessary when defining time summaries or message type mapping for little_r data. The contents of the default ASCII2NC configuration file are described below. -__________________ +_____________________ .. code-block:: none @@ -558,7 +559,7 @@ __________________ The configuration options listed above are common to many MET tools and are described in :numref:`config_options`. -_________________ +_____________________ .. code-block:: none @@ -567,7 +568,7 @@ _________________ The **time_summary** feature was implemented to allow additional processing of observations with high temporal resolution, such as SURFRAD data every 5 minutes. This option is described in :numref:`pb2nc configuration file`. -_________________ +_____________________ .. code-block:: none @@ -588,7 +589,7 @@ This entry is an array of dictionaries, each containing a **key** string and **v ascii2nc output -~~~~~~~~~~~~~~~ +--------------- The NetCDF output of the ASCII2NC tool is structured in the same way as the output of the PB2NC tool described in :numref:`pb2nc output`. @@ -596,14 +597,14 @@ The NetCDF output of the ASCII2NC tool is structured in the same way as the outp MADIS2NC tool -_____________ +============= This section describes how to run the MADIS2NC tool. The MADIS2NC tool is used to reformat `Meteorological Assimilation Data Ingest System (MADIS) `_ point observations into the NetCDF format expected by the MET statistics tools. An optional configuration file controls the processing of the point observations. The MADIS2NC tool supports many of the MADIS data types, as listed in the usage statement below. Support for additional MADIS data types may be added in the future based on user feedback. madis2nc usage -~~~~~~~~~~~~~~ +-------------- The usage statement for the MADIS2NC tool is shown below: @@ -686,7 +687,7 @@ In this example, the MADIS2NC tool will reformat the input sample_madis_obs.nc f madis2nc configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- The default configuration file for the MADIS2NC tool named **Madis2NcConfig_default** can be found in the installed *share/met/config* directory. It is recommended that users make a copy of this file prior to modifying its contents. @@ -694,8 +695,7 @@ The default configuration file for the MADIS2NC tool named **Madis2NcConfig_defa The MADIS2NC configuration file is optional and only necessary when defining time summaries. The contents of the default MADIS2NC configuration file are described below. -_________________ - +_____________________ .. code-block:: none @@ -704,8 +704,7 @@ _________________ The configuration options listed above are common to many MET tools and are described in :numref:`config_options`. -__________________ - +_____________________ .. code-block:: none @@ -716,7 +715,7 @@ The **time_summary** dictionary is described in :numref:`pb2nc configuration fil madis2nc output -~~~~~~~~~~~~~~~ +--------------- The NetCDF output of the MADIS2NC tool is structured in the same way as the output of the PB2NC tool described in :numref:`pb2nc output`. @@ -724,14 +723,14 @@ The NetCDF output of the MADIS2NC tool is structured in the same way as the outp LIDAR2NC tool -_____________ +============= The LIDAR2NC tool creates a NetCDF point observation file from a CALIPSO HDF data file. Not all of the data present in the CALIPSO file is reproduced in the output, however. Instead, the output focuses mostly on information about clouds (as opposed to aerosols) as seen by the satellite along its ground track. lidar2nc usage -~~~~~~~~~~~~~~ +-------------- The usage statement for LIDAR2NC tool is shown below: @@ -767,7 +766,7 @@ Optional arguments for lidar2nc 5. The **-compress level** option indicates the desired level of compression (deflate level) for NetCDF variables. The valid level is between 0 and 9. The value of "level" will override the default setting of 0 from the configuration file or the environment variable MET_NC_COMPRESS. Setting the compression level to 0 will make no compression for the NetCDF output. Lower number is for fast compression and higher number is for better compression. lidar2nc output -~~~~~~~~~~~~~~~ +--------------- Each observation type in the lidar2nc output is assigned a GRIB code. These are outlined in :numref:`lidar2nc_grib_code_table`. GRIB codes were assigned to these fields arbitrarily, with GRIB codes in the 600s denoting individual bit fields taken from the feature classification flag field in the CALIPSO file. @@ -839,14 +838,14 @@ We will not give a detailed description of each CALIPSO data product that lidar2 IODA2NC tool -____________ +============ This section describes the IODA2NC tool which is used to reformat IODA (Interface for Observation Data Access) point observations from the `Joint Center for Satellite Data Assimilation (JCSDA) `_ into the NetCDF format expected by the MET statistics tools. An optional configuration file controls the processing of the point observations. The IODA2NC tool reads NetCDF point observation files created by the `IODA Converters `_. Support for interfacing with data from IODA may be added in the future based on user feedback. ioda2nc usage -~~~~~~~~~~~~~ +------------- The usage statement for the IODA2NC tool is shown below: @@ -905,13 +904,13 @@ In this example, the IODA2NC tool will reformat the data in the input ioda.NC001 ioda2nc configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------- The default configuration file for the IODA2NC tool named **IODA2NcConfig_default** can be found in the installed *share/met/config* directory. It is recommended that users make a copy of this file prior to modifying its contents. The IODA2NC configuration file is optional and only necessary when defining filtering the input observations or defining time summaries. The contents of the default IODA2NC configuration file are described below. -__________________ +_____________________ .. code-block:: none @@ -922,8 +921,7 @@ __________________ The configuration options listed above are common to many MET tools and are described in :numref:`config_options`. -_________________ - +_____________________ .. code-block:: none @@ -939,7 +937,7 @@ _________________ The configuration options listed above are supported by other point observation pre-processing tools and are described in :numref:`pb2nc configuration file`. -_________________ +_____________________ .. code-block:: none @@ -947,7 +945,7 @@ _________________ This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of input IODA variable names to output variable names. The default IODA map, obs_var_map, is appended to this map. -_________________ +_____________________ .. code-block:: none @@ -961,7 +959,7 @@ _________________ This entry is an array of dictionaries, each containing a **key** string and **val** string which define a mapping of metadata for IODA data files. -_________________ +_____________________ .. code-block:: none @@ -971,13 +969,13 @@ The **missing_thresh** option is an array of thresholds. Any data values which m ioda2nc output -~~~~~~~~~~~~~~ +-------------- The NetCDF output of the IODA2NC tool is structured in the same way as the output of the PB2NC tool described in :numref:`pb2nc output`. Point2Grid tool -_______________ +=============== The Point2Grid tool takes point observations from a NetCDF output file from one of the four previously mentioned MET tools (ascii2nc, madis2nc, pb2nc, lidar2nc) and creates a gridded NetCDF file. The other point observations are GOES-16/17 input files in NetCDF format (especially, Aerosol Optical Depth. Future development will include support for reading input files not produced from MET tools. @@ -985,7 +983,7 @@ MET version 10.1 adds support for the passing point observations to point2grid u point2grid usage -~~~~~~~~~~~~~~~~ +---------------- The usage statement for the Point2Grid tool is shown below: @@ -1085,7 +1083,7 @@ Please refer to :numref:`Appendix F, Section %s ` for more details ab point2grid output -~~~~~~~~~~~~~~~~~ +----------------- The point2grid tool will output a gridded NetCDF file containing the following: @@ -1113,15 +1111,14 @@ The point2grid tool will output a gridded NetCDF file containing the following: For MET observation input and CF complaint NetCDF input with 2D time variable: The latest observation time within the target grid is saved as the observation time. If the "valid_time" is configured at the configuration file, the valid_time from the configuration file is saved into the output file. point2grid configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------- The default configuration file for the point2grid tool named **Point2GridConfig_default** can be found in the installed *share/met/config* directory. It is recommended that users make a copy of this file prior to modifying its contents. The point2grid configuration file is optional and only necessary when defining the variable name instead of GRIB code or filtering by time. The contents of the default MADIS2NC configuration file are described below. -_______________ - +_____________________ .. code-block:: none @@ -1130,8 +1127,7 @@ _______________ The configuration options listed above are common to many MET tools and are described in :numref:`config_options`. -__________________ - +_____________________ .. code-block:: none diff --git a/met/docs/Users_Guide/refs.rst b/met/docs/Users_Guide/refs.rst index 647e3591bf..423be1a479 100644 --- a/met/docs/Users_Guide/refs.rst +++ b/met/docs/Users_Guide/refs.rst @@ -1,7 +1,8 @@ .. _refs: +********** References -========== +********** .. _Aberson-1998: diff --git a/met/docs/Users_Guide/release-notes.rst b/met/docs/Users_Guide/release-notes.rst index 5cac6e7006..1f11648ec9 100644 --- a/met/docs/Users_Guide/release-notes.rst +++ b/met/docs/Users_Guide/release-notes.rst @@ -1,12 +1,12 @@ -MET release notes -_________________ +MET Release Notes +================= When applicable, release notes are followed by the GitHub issue number which describes the bugfix, enhancement, or new feature: `MET GitHub issues. `_ MET Version 10.1.0-beta5 release notes (20220114) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------- * Enhancements: @@ -27,7 +27,7 @@ MET Version 10.1.0-beta5 release notes (20220114) * Fix IODA2NC to handle the same input file being provided multiple times (`#1965 `_). MET Version 10.1.0-beta4 release notes (20211117) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------- * Enhancements: @@ -48,7 +48,7 @@ MET Version 10.1.0-beta4 release notes (20211117) * Enhance the documentation with meta-data that is expected by MET for netCDF (`#1949 `_). MET Version 10.1.0-beta3 release notes (20211006) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------- * New tools: @@ -79,7 +79,7 @@ MET Version 10.1.0-beta3 release notes (20211006) * Reduce the Security hotspots from SonarQube (`#1903 `_). MET Version 10.1.0-beta2 release notes (20210901) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------- * New output: @@ -116,7 +116,7 @@ MET Version 10.1.0-beta2 release notes (20210901) * Update FAQ in User's Guide with info from webpage FAQ (`#1834 `_). MET Version 10.1.0-beta1 release notes (20210613) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------- * Add the HSS_EC statistic to the MCTS line type and a configurable option for its computation (`#1749 `_). * Implement a common API for reading and writing the common NetCDF point observation file format (`#1402 `_ and `#1581 `_). @@ -127,7 +127,7 @@ MET Version 10.1.0-beta1 release notes (20210613) * Add anchors to link directly to configuration items in the MET User's Guide (`#1811 `_). MET Version 10.0.0 release notes (20210510) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------- * Repository and build: diff --git a/met/docs/Users_Guide/rmw-analysis.rst b/met/docs/Users_Guide/rmw-analysis.rst index 9ae27fe82f..6b0a849de9 100644 --- a/met/docs/Users_Guide/rmw-analysis.rst +++ b/met/docs/Users_Guide/rmw-analysis.rst @@ -1,18 +1,19 @@ .. _rmw-analysis: +***************** RMW-Analysis Tool -================= +***************** Introduction -____________ +============ The RMW-Analysis tool analyzes a set of output files generated by the TC-RMW tool. For each grid cell it aggregates variable statistics across the set and across the track points of the tc_rmw output files. The statistics are mean, standard deviation, minimum and maximum. Note that tc_rmw should be set to use the same scale factor of the radius of maximum winds (RMW) as the unit of range for its range-azimuth grid. The specified data variables on the range-azimuth-vertical grid then share a common range scale of RMW before aggregation by rmw_analysis. Practical information -_____________________ +===================== rmw_analysis usage -~~~~~~~~~~~~~~~~~~ +------------------ The following sections describe the usage statement, required arguments, and optional arguments for **rmw_analysis**. @@ -46,7 +47,7 @@ Optional arguments for rmw_analysis 5. The **-v level** option indicates the desired level of verbosity. The contents of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity above 1 will increase the amount of logging. rmw_analysis configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- The default configuration file for the RMW-Analysis tool named **RMWAnalysisConfig_default** can be found in the installed *share/met/config/* directory. It is encouraged for users to copy these default files before modifying their contents. The contents of the configuration file are described in the subsections below. @@ -86,6 +87,6 @@ The track filter options available in rmw_analysis and listed above are describe rmw_analysis output file -~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------ The NetCDF output file will inherit the spatial grid from the first tc_rmw output file in the output file list. All tc_rmw files in this list must have the same grid dimension sizes. A NetCDF output error will result if that is not the case. For each data variable specified in the config file, four corresponding NetCDF variables will be written, e.g. TMP_mean, TMP_stdev, TMP_min, TMP_max. No track point dimension is retained in the rmw_analysis output. diff --git a/met/docs/Users_Guide/series-analysis.rst b/met/docs/Users_Guide/series-analysis.rst index 640d948012..95766ae35b 100644 --- a/met/docs/Users_Guide/series-analysis.rst +++ b/met/docs/Users_Guide/series-analysis.rst @@ -1,15 +1,16 @@ .. _series-analysis: +******************** Series-Analysis Tool -==================== +******************** Introduction -____________ +============ The Series-Analysis Tool accumulates statistics separately for each horizontal grid location over a series. Often, this series is over time or height, though any type of series is possible. This differs from the Grid-Stat tool in that Grid-Stat verifies all grid locations together as a group. Thus, the Series-Analysis Tool can be used to find verification information specific to certain locations or see how model performance varies over the domain. Practical Information -_____________________ +===================== This Series-Analysis tool performs verification of gridded model fields using matching gridded observation fields. It computes a variety of user-selected statistics. These statistics are a subset of those produced by the Grid-Stat tool, with options for statistic types, thresholds, and conditional verification options as discussed in :numref:`grid-stat`. However, these statistics are computed separately for each grid location and accumulated over some series such as time or height, rather than accumulated over the whole domain for a single time or height as is done by Grid-Stat. @@ -22,7 +23,7 @@ To define a time series of forecasts that all have the same valid time, set the To define a series of vertical levels all contained in a single input file, set the forecast and observation fields to a list of the vertical levels to be used. Pass the tool single forecast and observation files containing the vertical level data. The tool will loop over the forecast field entries, extract that field from the input forecast file, and then search the observation file for a matching record. series_analysis usage -~~~~~~~~~~~~~~~~~~~~~ +--------------------- The usage statement for the Series-Analysis tool is shown below: @@ -78,7 +79,7 @@ An example of the series_analysis calling sequence is shown below: In this example, the Series-Analysis tool will process the list of forecast and observation files specified in the text file lists into statistics for each grid location using settings specified in the configuration file. Series-Analysis will create an output NetCDF file containing requested statistics. series_analysis output -~~~~~~~~~~~~~~~~~~~~~~ +---------------------- The Series-Analysis tool produces NetCDF files containing output statistics for each grid location from the input files. The details about the output statistics available from each output line type are detailed in Chapter 5 since they are also produced by the Grid-Stat Tool. A subset of these can be produced by this tool, with the most notable exceptions being the wind vector and neighborhood statistics. Users can inventory the contents of the Series-Analysis output files using the ncdump -h command to view header information. Additionally, ncview or the Plot-Data-Plane tool can be used to visualize the output. An example of Series-Analysis output is shown in :numref:`series-analysis_Glibert_precip` below. @@ -89,7 +90,7 @@ The Series-Analysis tool produces NetCDF files containing output statistics for An example of the Gilbert Skill Score for precipitation forecasts at each grid location for a month of files. series_analysis configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------- The default configuration file for the Series-Analysis tool named **SeriesAnalysisConfig_default** can be found in the installed *share/met/config* directory. The contents of the configuration file are described in the subsections below. Note that environment variables may be used when editing configuration files, as described in the :numref:`pb2nc configuration file` for the PB2NC tool. diff --git a/met/docs/Users_Guide/stat-analysis.rst b/met/docs/Users_Guide/stat-analysis.rst index 80c843b9a3..b398827234 100644 --- a/met/docs/Users_Guide/stat-analysis.rst +++ b/met/docs/Users_Guide/stat-analysis.rst @@ -1,28 +1,29 @@ .. _stat-analysis: +****************** Stat-Analysis Tool -================== +****************** Introduction -____________ +============ The Stat-Analysis tool ties together results from the Point-Stat, Grid-Stat, Ensemble-Stat, Wavelet-Stat, and TC-Gen tools by providing summary statistical information and a way to filter their STAT output files. It processes the STAT output created by the other MET tools in a variety of ways which are described in this section. MET version 9.0 adds support for the passing matched pair data (MPR) into Stat-Analysis using a Python script with the "-lookin python ..." option. An example of running Stat-Analysis with Python embedding is shown in :numref:`stat_analysis-usage`. Scientific and statistical aspects -__________________________________ +================================== The Stat-Analysis tool can perform a variety of analyses, and each type of analysis is called a "job". The job types include the ability to (i) aggregate results over a user-specified time; (ii) stratify statistics based on time of day, model initialization time, lead-time, model run identifier, output filename, or wavelet decomposition scale; and (iii) compute specific verification indices such as the GO Index [1]_ and wind direction statistics. Future functionality may include information about time-trends and/or calculations based on climatology (e.g., anomaly correlation). This section summarizes the capabilities of the supported Stat-Analysis jobs. Filter STAT lines -~~~~~~~~~~~~~~~~~ +----------------- The Stat-Analysis "filter" job simply filters out specific STAT lines based on user-specified search criteria. All of the STAT lines that are retained from one or many files are written to a single output file. The output file for filtered STAT lines must be specified using the **-dump_row** job command option. Summary statistics for columns -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ The Stat-Analysis "summary" job produces summary information for columns of data. After the user specifies the column(s) of interest and any other relevant search criteria, summary information is produced from values in those column(s) of data. The summary statistics produced are: mean, standard deviation, minimum, maximum, the 10th, 25th, 50th, 75th, and 90th percentiles, the interquartile range, the range, and both weighted and unweighted means using the logic prescribed by the World Meteorological Organization (WMO). @@ -37,12 +38,12 @@ The **-derive** job command option can be used to perform the derivation of stat .. _StA_Aggregated-values-from: Aggregated values from multiple STAT lines -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------ The Stat-Analysis "aggregate" job aggregates values from multiple STAT lines of the same type. The user may specify the specific line type of interest and any other relevant search criteria. The Stat-Analysis tool then creates sums of each of the values in all lines matching the search criteria. The aggregated data are output as the same line type as the user specified. The STAT line types which may be aggregated in this way are the contingency table (FHO, CTC, PCT, MCTC, NBRCTC), partial sums (SL1L2, SAL1L2, VL1L2, and VAL1L2), and other (ISC, ECNT, RPS, RHIST, PHIST, RELP, NBRCNT, SSVAR, and GRAD) line types. Aggregate STAT lines and produce aggregated statistics -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------------ The Stat-Analysis "aggregate-stat" job aggregates multiple STAT lines of the same type together and produces relevant statistics from the aggregated line. This may be done in the same manner listed above in :numref:`StA_Aggregated-values-from`. However, rather than writing out the aggregated STAT line itself, the relevant statistics generated from that aggregated line are provided in the output. Specifically, if a contingency table line type (FHO, CTC, PCT, MCTC, or NBRCTC) has been aggregated, contingency table statistics (CTS, ECLV, PSTD, MCTS, or NBRCTS) line types can be computed. If a partial sums line type (SL1L2 or SAL1L2) has been aggregated, the continuous statistics (CNT) line type can be computed. If a vector partial sums line type (VL1L2) has been aggregated, the vector continuous statistics (VCNT) line type can be computed. For ensembles, the ORANK line type can be accumulated into ECNT, RPS, RHIST, PHIST, RELP, or SSVAR output. If the matched pair line type (MPR) has been aggregated, may output line types (FHO, CTC, CTS, CNT, MCTC, MCTS, SL1L2, SAL1L2, VL1L2, VCNT, WDIR, PCT, PSTD, PJC, PRC, or ECLV) can be computed. Multiple output line types may be specified for each "aggregate-stat" job, as long as each output is derivable from the input. @@ -53,7 +54,7 @@ When aggregating the matched pair line type (MPR) and computing an output contin .. _StA_Skill-Score-Index: Skill Score Index -~~~~~~~~~~~~~~~~~ +----------------- The Stat-Analysis "ss_index", "go_index", and "cbs_index" jobs calculate skill score indices by weighting scores for meteorological fields at different levels and lead times. Pre-defined configuration files are provided for the GO Index and CBS Index which are special cases of the highly configurable skill score index job. @@ -92,7 +93,7 @@ When running a skill score index job using the "-out_stat" job command option, a .. _StA_Go-Index: GO Index -~~~~~~~~ +-------- The "go_index" job is a special case of the "ss_index" job, described in :numref:`StA_Skill-Score-Index`. The GO Index is a weighted average of 48 skill scores of RMSE statistics for wind speed, dew point temperature, temperature, height, and pressure at several levels in the atmosphere. The variables, levels, and lead times included in the index are listed in :numref:`compute_GO_Index` and are defined by the default "STATAnalysisConfig_GO_Index" configuration file. The partial sums (SL1L2 lines in the STAT output) for each of these variables at each level and lead time must have been computed in a previous step. The Stat-Analysis tool then uses the weights in :numref:`compute_GO_Index` to compute values for the GO Index. @@ -190,7 +191,7 @@ The "go_index" job is a special case of the "ss_index" job, described in :numref .. _StA_CBS-Index: CBS Index -~~~~~~~~~ +--------- The "cbs_index" job is a special case of the "ss_index" job, described in :numref:`StA_Skill-Score-Index`. The CBS Index is a weighted average of 40 skill scores of RMSE statistics for mean sea level pressure, height, and wind speed at multiple levels computed over the northern hemisphere, southern hemisphere and the tropics. The variables, levels, lead times, and regions included in the index are listed in :numref:`compute_CBS_Index` and are defined by the default "STATAnalysisConfig_CBS_Index" configuration file. The partial sums (SL1L2 lines in the STAT output) for each of these variables for each level, lead time, and masking region must have been computed in a previous step. The Stat-Analysis tool then uses the weights in :numref:`compute_CBS_Index` to compute values for the CBS Index. @@ -232,12 +233,12 @@ The "cbs_index" job is a special case of the "ss_index" job, described in :numre - x Ramp Events -~~~~~~~~~~~ +----------- The Stat-Analysis "ramp" job identifies ramp events (large increases or decreases in values over a time window) in both the forecast and observation data. It categorizes these events as hits, misses, false alarms, or correct negatives by applying a configurable matching time window and computes the corresponding categorical statistics. Wind Direction Statistics -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- The Stat-Analysis "aggregate_stat" job can read vector partial sums and derive wind direction error statistics (WDIR). The vector partial sums (VL1L2 or VAL1L2) or matched pairs (MPR) for the UGRD and VGRD must have been computed in a previous step, i.e. by Point-Stat or Grid-Stat tools. This job computes an average forecast wind direction and an average observed wind direction along with their difference. The output is in degrees. In Point-Stat and Grid-Stat, the UGRD and VGRD can be verified using thresholds on their values or on the calculated wind speed. If thresholds have been applied, the wind direction statistics are calculated for each threshold. @@ -258,14 +259,14 @@ Once the appropriate lines have been generated for each verification time of int 2. For the "AGGR_WDIR" line, the input VL1L2 lines are first aggregated into a single line of partial sums where the weight for each line is determined by the number of points it represents. From this aggregated line, the mean forecast wind direction, observation wind direction, and the associated error are computed and written out. Practical information -_____________________ +===================== The following sections describe the usage statement, required arguments and optional arguments for the Stat-Analysis tool. .. _stat_analysis-usage: stat_analysis usage -~~~~~~~~~~~~~~~~~~~ +------------------- The usage statement for the Stat-Analysis tool is shown below: @@ -331,7 +332,7 @@ In this example, rather than passing the MPR output lines from Point-Stat direct .. _stat_analysis-configuration-file: stat_analysis configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------- The default configuration file for the Stat-Analysis tool named **STATAnalysisConfig_default** can be found in the installed *share/met/config* directory. The version used for the example run in :numref:`installation` is also available in *scripts/config*. Like the other configuration files described in this document, it is recommended that users make a copy of these files prior to modifying their contents. @@ -692,7 +693,7 @@ These job command options are analogous to the options listed above but apply wh When processing input ORANK lines and writing output RHIST or PHIST lines, this option defines the output histogram bin width to be used. stat-analysis tool output -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- The output generated by the Stat-Analysis tool contains statistics produced by the analysis. It also records information about the analysis job that produced the output for each line. Generally, the output is printed to the screen. However, it can be redirected to an output file using the "**-out**" option. The format of output from each STAT job command is described below. diff --git a/met/docs/Users_Guide/tc-dland.rst b/met/docs/Users_Guide/tc-dland.rst index 163c732968..078d4d3a9e 100644 --- a/met/docs/Users_Guide/tc-dland.rst +++ b/met/docs/Users_Guide/tc-dland.rst @@ -1,17 +1,18 @@ .. _tc-dland: +************* TC-Dland Tool -============= +************* Introduction -____________ +============ Many filtering criteria within the MET-TC tools depend on the distinction between when a storm is over land or water. The TC-dland tool was developed to aid in quickly parsing data for filter jobs that only verify over water, threshold verification based on distance to land, and exclusion of forecasts outside a specified time window of landfall. For each grid point in the user-specified grid, it computes the great circle arc distance to the nearest coast line. Compared to the simple Euclidean distances, great circle arc distances are more accurate but take considerably longer to compute. Grid points over water have distances greater than zero while points over land have distances less than zero. While the TC-dland tool is available to be run, most users will find the precomputed distance to land files distributed with the release sufficient. Therefore, the typical user will not actually need to run this tool. Input/output format -___________________ +=================== The input for the TC-dland tool is a file containing the longitude (degrees east) and latitude (degrees north) of all the coastlines and islands considered to be a significant landmass. The default input is to use all three land data files (**aland.dat, shland.dat, wland.dat**) found in the installed *share/met/tc_data/* directory. The use of all three files produces a global land data file. The **aland.dat** file contains the longitude and latitude distinctions used by NHC for the Atlantic and eastern North Pacific basins, the **shland.dat** contains longitude and latitude distinctions for the Southern Hemisphere (south Pacific and South Indian Ocean), and the **wland.dat** contains the remainder of the Northern Hemisphere (western North Pacific and North Indian Ocean). Users may supply their own input file in order to refine the definition of coastlines and a significant landmass. @@ -22,12 +23,12 @@ The output file from TC-dland is a NetCDF format file containing a gridded field **dland_global_tenth_degree.nc:** TC-dland output from all three land data files (global coverage) using a 1/10th degree grid. Practical information -_____________________ +===================== This section briefly describes how to run tc_dland. The default grid is set to 1/10th degree Northwest (NW) hemispheric quadrant (over North America) grid. tc_dland usage -~~~~~~~~~~~~~~ +-------------- .. code-block:: none diff --git a/met/docs/Users_Guide/tc-gen.rst b/met/docs/Users_Guide/tc-gen.rst index bf91572fcc..d69036a603 100644 --- a/met/docs/Users_Guide/tc-gen.rst +++ b/met/docs/Users_Guide/tc-gen.rst @@ -1,17 +1,18 @@ .. _tc-gen: +*********** TC-Gen Tool -=========== +*********** Introduction -____________ +============ The TC-Gen tool provides verification of deterministic and probabilistic tropical cyclone genesis forecasts in the ATCF file and shapefile formats. Producing reliable tropical cyclone genesis forecasts is an important metric for global numerical weather prediction models. This tool ingests deterministic model output post-processed by genesis tracking software (e.g. GFDL vortex tracker), ATCF edeck files containing probability of genesis forecasts, operational shapefile warning areas, and ATCF reference track dataset(s) (e.g. Best Track analysis and CARQ operational tracks). It writes categorical counts and statistics. The capability to modify the spatial and temporal tolerances when matching forecasts to reference genesis events, as well as scoring those matched pairs, gives users the ability to condition the criteria based on model performance and/or conduct sensitivity analyses. Statistical aspects are outlined in :numref:`tc-gen_stat_aspects` and practical aspects of the TC-Gen tool are described in :numref:`tc-gen_practical_info`. .. _tc-gen_stat_aspects: Statistical aspects -___________________ +=================== The TC-Gen tool processes both deterministic and probabilistic forecasts. @@ -32,12 +33,12 @@ Care should be taken when interpreting the statistics for filtered data. In some .. _tc-gen_practical_info: Practical information -_____________________ +===================== This section describes how to configure and run the TC-Gen tool. The following sections describe the usage statement, required arguments, and optional arguments for tc_gen. tc_gen usage -~~~~~~~~~~~~ +------------ The usage statement for tc_gen is shown below: @@ -157,7 +158,7 @@ The TC-Gen tool implements the following logic: * Report the Nx2 probabilistic contingency table counts and statistics for each lead time. These counts and statistics are identified in the output files as *GENESIS_SHAPE*. tc_gen configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- The default configuration file for the **TC-Gen** tool named **TCGenConfig_default** can be found in the installed *share/met/config* directory. Like the other configuration files described in this document, it is recommended that users make a copy of these files prior to modifying their contents. @@ -481,7 +482,7 @@ ______________________ The configuration options listed above are common to many MET tools and are described in :numref:`config_options`. TC-Gen writes output for 2x2 contingency tables to the **FHO**, **CTC**, and **CTS** line types when verifying deterministic genesis forecasts specified using the **-track** command line option. TC-Gen writes output for Nx2 probabilistic contingency tables to the **PCT**, **PSTD**, **PJC**, and **PRC** line types when verifying the probability of genesis forecasts specified using the **-edeck** command line option and probabilistic shapefiles using the **-shape** command line option. Note that the **genmpr** line type is specific to TC-Gen and describes individual genesis matched pairs. tc_gen output -~~~~~~~~~~~~~ +------------- TC-Gen produces output in STAT and, optionally, ASCII and NetCDF formats. The ASCII output duplicates the STAT output but has the data organized by line type. The output files are created based on the **-out** command line argument. The default output base name, **./tc_gen** writes output files in the current working directory named **tc_gen.stat** and, optionally, **tc_gen_pairs.nc** and **tc_gen_{TYPE}.txt** for each of the supported output line types. These output files can easily be redirected to another location using the **-out** command line option. The format of the STAT and ASCII output of the TC-Gen tool matches the output of other MET tools with the exception of the genesis matched pair line type. Please refer to the tables in :numref:`point_stat-output` for a description of the common output line types. The genesis matched pair line type and NetCDF output file are described below. diff --git a/met/docs/Users_Guide/tc-pairs.rst b/met/docs/Users_Guide/tc-pairs.rst index b32cbc15b1..3de6d0b470 100644 --- a/met/docs/Users_Guide/tc-pairs.rst +++ b/met/docs/Users_Guide/tc-pairs.rst @@ -1,22 +1,23 @@ .. _tc-pairs: +************* TC-Pairs Tool -============= +************* Introduction -____________ +============ The TC-Pairs tool provides verification for tropical cyclone forecasts in ATCF file format. It matches an ATCF format tropical cyclone (TC) forecast with a second ATCF format reference TC dataset (most commonly the Best Track analysis). The TC-Pairs tool processes both track and intensity adeck data and probabilistic edeck data. The adeck matched pairs contain position errors, as well as wind, sea level pressure, and distance to land values for each TC dataset. The edeck matched pairs contain probabilistic forecast values and the verifying observation values. The pair generation can be subset based on user-defined filtering criteria. Practical aspects of the TC-Pairs tool are described in :numref:`TC-Pairs_Practical-information`. .. _TC-Pairs_Practical-information: Practical information -_____________________ +===================== This section describes how to configure and run the TC-Pairs tool. The TC-Pairs tool is used to match a tropical cyclone model forecast to a corresponding reference dataset. Both tropical cyclone forecast/reference data must be in ATCF format. Output from the TC-dland tool (NetCDF gridded distance file) is also a required input for the TC-Pairs tool. It is recommended to run tc_pairs on a storm-by-storm basis, rather than over multiple storms or seasons to avoid memory issues. tc_pairs usage -~~~~~~~~~~~~~~ +-------------- The usage statement for tc_pairs is shown below: @@ -79,7 +80,7 @@ The TC-Pairs tool implements the following logic: • For each edeck/bdeck pair, write paired edeck probabilities and matching bdeck values to output PROBRIRW lines. tc_pairs configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- The default configuration file for the TC-Pairs tool named **TCPairsConfig_default** can be found in the installed *share/met/config/* directory. Users are encouraged to copy these default files before modifying their contents. The contents of the configuration file are described in the subsections below. @@ -271,7 +272,7 @@ parameter will result in missed matches. .. _tc_pairs-output: tc_pairs output -~~~~~~~~~~~~~~~ +--------------- TC-Pairs produces output in TCST format. The default output file name can be overwritten using the -out file argument in the usage statement. The TCST file output from TC-Pairs may be used as input into the TC-Stat tool. The header column in the TC-Pairs output is described in :numref:`TCST Header`. diff --git a/met/docs/Users_Guide/tc-rmw.rst b/met/docs/Users_Guide/tc-rmw.rst index 852973b0f1..8e11d70798 100644 --- a/met/docs/Users_Guide/tc-rmw.rst +++ b/met/docs/Users_Guide/tc-rmw.rst @@ -1,18 +1,19 @@ .. _tc-rmw: +*********** TC-RMW Tool -=========== +*********** Introduction -____________ +============ The TC-RMW tool regrids tropical cyclone model data onto a moving range-azimuth grid centered on points along the storm track provided in ATCF format, most likely the adeck generated from the file. The radial grid spacing may be set as a factor of the radius of maximum winds (RMW). If wind fields are specified in the configuration file the radial and tangential wind components will be computed. Any regridding method available in MET can be used to interpolate data on the model output grid to the specified range-azimuth grid. The regridding will be done separately on each vertical level. The model data files must coincide with track points in a user provided ATCF formatted track file. Practical information -_____________________ +===================== tc_rmw usage -~~~~~~~~~~~~ +------------ The following sections describe the usage statement, required arguments, and optional arguments for tc_rmw. @@ -47,7 +48,7 @@ Optional arguments for tc_rmw 6. The **-v level** option indicates the desired level of verbosity. The contents of "level" will override the default setting of 2. Setting the verbosity to 0 will make the tool run with no log messages, while increasing the verbosity above 1 will increase the amount of logging. tc_rmw configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- The default configuration file for the TC-RMW tool named **TCRMWConfig_default** can be found in the installed *share/met/config/* directory. It is encouraged for users to copy these default files before modifying their contents. The contents of the configuration file are described in the subsections below. @@ -123,7 +124,7 @@ _______________________ The **rmw_scale** parameter overrides the **max_range_km** parameter. When this is set the radial grid spacing will be **rmw_scale** in units of the RMW, which varies along the storm track. tc_rmw output file -~~~~~~~~~~~~~~~~~~ +------------------ The NetCDF output file contains the following dimensions: diff --git a/met/docs/Users_Guide/tc-stat.rst b/met/docs/Users_Guide/tc-stat.rst index 7052e80958..6db00f3160 100644 --- a/met/docs/Users_Guide/tc-stat.rst +++ b/met/docs/Users_Guide/tc-stat.rst @@ -1,27 +1,28 @@ .. _tc-stat: +************ TC-Stat Tool -============ +************ Introduction -____________ +============ The TC-Stat tool ties together results from the TC-Pairs tool by providing summary statistics and filtering jobs on TCST output files. The TC-Stat tool requires TCST output from the TC-Pairs tool. See :numref:`tc_stat-output` of this user's guide for information on the TCST output format of the TC-Pairs tool. The TC-Stat tool supports several analysis job types. The **filter** job stratifies the TCST data using various conditions and thresholds described in :numref:`tc_stat-configuration-file`. The **summary** job produces summary statistics including frequency of superior performance, time-series independence calculations, and confidence intervals on the mean. The **rirw** job processes TCMPR lines, identifies adeck and bdeck rapid intensification or weakening events, populates a 2x2 contingency table, and derives contingency table statistics. The **probrirw job** processes PROBRIRW lines, populates an Nx2 probabilistic contingency table, and derives probabilistic statistics. The statistical aspects are described in :numref:`Statistical-aspects`, and practical use information for the TC-Stat tool is described in :numref:`Practical-information-1`. .. _Statistical-aspects: Statistical aspects -___________________ +=================== Filter TCST lines -~~~~~~~~~~~~~~~~~ +----------------- The TC-Stat tool can be used to simply filter specific lines of the TCST file based on user-defined filtering criteria. All of the TCST lines that are retained from one or more files are written out to a single output file. The output file is also in TCST format. Filtering options are outlined below in :numref:`tc_stat-configuration-file` (configuration file). If multiple filtering options are listed, the job will be performed on their intersection. Summary statistics for columns -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ The TC-Stat tool can be used to produce summary information for a single column of data. After the user specifies the specific column of interest, and any other relevant search criteria, summary information is produced from values in that column of data. The summary statistics produced are listed in :numref:`table_columnar_output_summary_tc_stat`. @@ -52,14 +53,14 @@ Time-Series Independence The time-series independence evaluates effective forecast separation time using the Siegel method, by comparing the number of runs above and below the mean error to an expected value. This calculation expects the columns in the summary job to be a time series. The output includes the forecast hour interval and the number of hours to independence. Rapid Intensification/Weakening -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- The TC-Stat tool can be used to read TCMPR lines and compare the occurrence of rapid intensification (i.e. increase in intensity) or weakening (i.e. decrease in intensity) between the adeck and bdeck. The rapid intensification or weakening is defined by the change of maximum wind speed (i.e. **AMAX_WIND** and **BMAX_WIND** columns) over a specified amount of time. Accurately forecasting large changes in intensity is a challenging problem and this job helps quantify a model's ability to do so. Users may specify several job command options to configure the behavior of this job. Using these configurable options, the TC-Stat tool analyzes paired tracks and for each track point (i.e. each TCMPR line) determines whether rapid intensification or weakening occurred. For each point in time, it uses the forecast and BEST track event occurrence to populate a 2x2 contingency table. The job may be configured to require that forecast and BEST track events occur at exactly the same time to be considered a hit. Alternatively, the job may be configured to define a hit as long as the forecast and BEST track events occurred within a configurable time window. Using this relaxed matching criteria false alarms may be considered hits and misses may be considered correct negatives as long as the adeck and bdeck events were close enough in time. Each rirw job applies a single intensity change threshold. Therefore, assessing a model's performance with rapid intensification and weakening requires that two separate jobs be run. Probability of Rapid Intensification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------ The TC-Stat tool can be used to accumulate multiple PROBRIRW lines and derive probabilistic statistics summarizing performance. The PROBRIRW line contains a probabilistic forecast for a specified intensity change along with the actual intensity change that occurred in the BEST track. Accurately forecasting the likelihood of large changes in intensity is a challenging problem and this job helps quantify a model's ability to do so. @@ -68,12 +69,12 @@ Users may specify several job command options to configure the behavior of this .. _Practical-information-1: Practical information -_____________________ +===================== The following sections describe the usage statement, required arguments, and optional arguments for tc_stat. tc_stat usage -~~~~~~~~~~~~~ +------------- The usage statement for tc_stat is shown below: @@ -370,7 +371,7 @@ _________________________ .. _tc_stat-output: tc_stat output -~~~~~~~~~~~~~~ +-------------- The output generated from the TC-Stat tool contains statistics produced by the analysis. Additionally, it includes information about the analysis job that produced the output for each line. The output can be redirected to an output file using the **-out** option. The format of output from each tc_stat job command is listed below. diff --git a/met/docs/Users_Guide/wavelet-stat.rst b/met/docs/Users_Guide/wavelet-stat.rst index 2b9a39291b..15d68616e5 100644 --- a/met/docs/Users_Guide/wavelet-stat.rst +++ b/met/docs/Users_Guide/wavelet-stat.rst @@ -1,12 +1,13 @@ .. _wavelet-stat: +***************** Wavelet-Stat Tool -================= +***************** .. _WS_Introduction: Introduction -____________ +============ The Wavelet-Stat tool decomposes two-dimensional forecasts and observations according to intensity and scale. This section describes the Wavelet-Stat tool, which enables users to apply the Intensity-Scale verification technique described by :ref:`Casati et al. (2004) `. @@ -21,10 +22,10 @@ The spatial scale components are obtained usually by applying a single band spat The Intensity-Scale technique evaluates the forecast skill as a function of the intensity values and of the spatial scale of the error. The scale components are obtained by applying a two dimensional Haar wavelet filter. Note that wavelets, because of their locality, are suitable for representing discontinuous fields characterized by few sparse non-zero features, such as precipitation. Moreover, the technique is based on a categorical approach, which is a robust and resistant approach, suitable for non-normally distributed variables, such as precipitation. The intensity-scale technique was specifically designed to cope with the difficult characteristics of precipitation fields, and for the verification of spatial precipitation forecasts. However, the intensity-scale technique can also be applied to verify other variables, such as cloud fraction. Scientific and statistical aspects -__________________________________ +================================== The method -~~~~~~~~~~ +---------- :ref:`Casati et al. (2004) ` applied the Intensity-Scale verification to preprocessed and re-calibrated (unbiased) data. The preprocessing was aimed to mainly normalize the data, and defined categorical thresholds so that each categorical bin had a similar sample size. The recalibration was performed to eliminate the forecast bias. Preprocessing and recalibration are not strictly necessary for the application of the Intensity-Scale technique. The MET Intensity-Scale Tool does not perform either, and applies the Intensity-Scale approach to biased forecasts, for categorical thresholds defined by the user. @@ -126,7 +127,7 @@ Note that the energy squared of the observation binary field is identical to the The spatial domain constraints -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------ The Intensity-Scale technique is constrained by the fact that orthogonal wavelets (discrete wavelet transforms) are usually performed dyadic domains, square domains of :math:`\mathbf{2^n} **x** :math:`\mathbf{2^n} grid-points. The Wavelet-Stat tool handles this issue based on settings in the configuration file by defining tiles of dimensions :math:`\mathbf{2^n} **x** :math:`\mathbf{2^n} over the input domain in the following ways: @@ -137,19 +138,19 @@ The Intensity-Scale technique is constrained by the fact that orthogonal wavelet 3. Padding: If the domain size is only slightly smaller than :math:`\mathbf{2^n} **x** :math:`\mathbf{2^n}, for certain variables (e.g. precipitation), it is advisable to expand the domain out to :math:`\mathbf{2^n} **x** :math:`\mathbf{2^n} grid-points by adding extra rows and/or columns of fill data. For precipitation variables, a fill value of zero is used. For continuous variables, such as temperature, the fill value is defined as the mean of the valid data in the rest of the field. A drawback to the padding method is the introduction of artificial data into the original field. Padding should only be used when a very small number of rows and/or columns need to be added. Aggregation of statistics on multiple cases -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------- The Stat-Analysis tool aggregates the intensity scale technique results. Since the results are scale-dependent, it is sensible to aggregate results from multiple model runs (e.g. daily runs for a season) on the same spatial domain, so that the scale components for each singular case will be the same number, and the domain, if not a square domain of :math:`\mathbf{2^n} **x** :math:`\mathbf{2^n} grid-points, will be treated in the same fashion. Similarly, the intensity thresholds for each run should all be the same. The MSE and forecast and observation squared energy for each scale and thresholds are aggregated simply with a weighted average, where weights are proportional to the number of grid-points used in each single run to evaluate the statistics. If the same domain is always used (and it should) the weights result all the same, and the weighted averaging is a simple mean. For each threshold, the aggregated Br is equal to the aggregated squared energy of the binary observation field, and the aggregated FBI is obtained as the ratio of the aggregated squared energies of the forecast and observation binary fields. From aggregated Br and FBI, the MSErandom for the aggregated runs can be evaluated using the same formula as for the single run. Finally, the Intensity-Scale Skill Score is evaluated by using the aggregated statistics within the same formula used for the single case. Practical information -_____________________ +===================== The following sections describe the usage statement, required arguments and optional arguments for the Stat-Analysis tool. wavelet_stat usage -~~~~~~~~~~~~~~~~~~ +------------------ The usage statement for the Wavelet-Stat tool is shown below: @@ -200,7 +201,7 @@ In the example, the Wavelet-Stat tool will verify the model data in the **sample .. _wavelet_stat-configuration-file: wavelet_stat configuration file -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- The default configuration file for the Wavelet-Stat tool, **WaveletStatConfig_default**, can be found in the installed *share/met/config* directory. Another version of the configuration file is provided in *scripts/config*. We recommend that users make a copy of the default (or other) configuration file prior to modifying it. The contents are described in more detail below. Note that environment variables may be used when editing configuration files, as described in the :numref:`pb2nc configuration file` for the PB2NC tool. @@ -311,7 +312,7 @@ The nc_pairs_flag is described in :numref:`grid_stat-configuration-file` .. _wavelet_stat-output: wavelet_stat output -~~~~~~~~~~~~~~~~~~~ +------------------- wavelet_stat produces output in STAT and, optionally, ASCII and NetCDF and PostScript formats. The ASCII output duplicates the STAT output but has the data organized by line type. While the Wavelet-Stat tool currently only outputs one STAT line type, additional line types may be added in future releases. The output files are written to the default output directory or the directory specified by the -outdir command line option. diff --git a/met/src/libcode/vx_data2d_nc_met/data2d_nc_met.cc b/met/src/libcode/vx_data2d_nc_met/data2d_nc_met.cc index 0bbb55c718..23c038e5d1 100644 --- a/met/src/libcode/vx_data2d_nc_met/data2d_nc_met.cc +++ b/met/src/libcode/vx_data2d_nc_met/data2d_nc_met.cc @@ -121,10 +121,7 @@ void MetNcMetDataFile::dump(ostream & out, int depth) const { //////////////////////////////////////////////////////////////////////// -bool MetNcMetDataFile::data_plane(VarInfo &vinfo, DataPlane &plane) - -{ - +bool MetNcMetDataFile::data_plane(VarInfo &vinfo, DataPlane &plane) { bool status = false; ConcatString req_time_str, data_time_str; VarInfoNcMet * vinfo_nc = (VarInfoNcMet *) &vinfo; @@ -221,14 +218,18 @@ int MetNcMetDataFile::data_plane_array(VarInfo &vinfo, //////////////////////////////////////////////////////////////////////// -int MetNcMetDataFile::index(VarInfo &vinfo){ +int MetNcMetDataFile::index(VarInfo &vinfo) { + + NcVarInfo *ncinfo = MetNc->find_var_name( vinfo.name().c_str() ); - if( NULL == MetNc->find_var_name( vinfo.name().c_str() ) ) return -1; + if( !ncinfo ) return(-1); - if( ( vinfo.valid() && MetNc->ValidTime != vinfo.valid() ) || - ( vinfo.init() && MetNc->InitTime != vinfo.init() ) || - ( !is_bad_data(vinfo.lead()) && MetNc->lead_time() != vinfo.lead() ) ) - return -1; + if( ( vinfo.valid() != 0 && ncinfo->ValidTime != vinfo.valid() ) || + ( vinfo.init() != 0 && ncinfo->InitTime != vinfo.init() ) || + ( !is_bad_data(vinfo.lead()) && ncinfo->lead_time() != vinfo.lead() ) ) + return(-1); - return 0; + return(0); } + +//////////////////////////////////////////////////////////////////////// diff --git a/met/src/libcode/vx_data2d_nc_met/data2d_nc_met.h b/met/src/libcode/vx_data2d_nc_met/data2d_nc_met.h index 0153d16467..eb72f67605 100644 --- a/met/src/libcode/vx_data2d_nc_met/data2d_nc_met.h +++ b/met/src/libcode/vx_data2d_nc_met/data2d_nc_met.h @@ -50,7 +50,6 @@ class MetNcMetDataFile : public Met2dDataFile { MetNcFile * MetNc; // allocated - // // set stuff // @@ -73,7 +72,6 @@ class MetNcMetDataFile : public Met2dDataFile { int index(VarInfo &); - // // do stuff // diff --git a/met/src/libcode/vx_data2d_nc_met/met_file.cc b/met/src/libcode/vx_data2d_nc_met/met_file.cc index 8abf239884..dd8f599c48 100644 --- a/met/src/libcode/vx_data2d_nc_met/met_file.cc +++ b/met/src/libcode/vx_data2d_nc_met/met_file.cc @@ -201,9 +201,6 @@ Nvars = 0; if ( Var ) { delete [] Var; Var = (NcVarInfo *) 0; } -ValidTime = (unixtime) 0; -InitTime = (unixtime) 0; - // // done // @@ -298,9 +295,8 @@ for (j=0; jvar, a, plane); // store the times // -plane.set_init ( ValidTime - lead_time() ); -plane.set_valid ( ValidTime ); -plane.set_lead ( lead_time() ); +plane.set_init ( info->InitTime ); +plane.set_valid ( info->ValidTime ); +plane.set_lead ( info->lead_time() ); plane.set_accum ( info->AccumTime ); // diff --git a/met/src/libcode/vx_data2d_nc_met/met_file.h b/met/src/libcode/vx_data2d_nc_met/met_file.h index 513cb0c49f..8510aa3159 100644 --- a/met/src/libcode/vx_data2d_nc_met/met_file.h +++ b/met/src/libcode/vx_data2d_nc_met/met_file.h @@ -59,20 +59,8 @@ class MetNcFile { void dump(ostream &, int = 0) const; - NcFile * Nc; // allocated - // - // time - // - - unixtime ValidTime; - - unixtime InitTime; - - int lead_time () const; // seconds - - // // dimensions // diff --git a/met/src/libcode/vx_nc_util/nc_var_info.cc b/met/src/libcode/vx_nc_util/nc_var_info.cc index b8acf6228f..d41d7c8762 100644 --- a/met/src/libcode/vx_nc_util/nc_var_info.cc +++ b/met/src/libcode/vx_nc_util/nc_var_info.cc @@ -58,7 +58,7 @@ unixtime get_att_value_unixtime(const NcAtt *att) { // - // Code for class NcNcVarInfo + // Code for class NcVarInfo // @@ -151,6 +151,10 @@ level_att.clear(); units_att.clear(); +ValidTime = (unixtime) 0; + +InitTime = (unixtime) 0; + AccumTime = 0; Ndims = 0; @@ -199,6 +203,12 @@ else out << "(nul)"; if ( units_att.length() > 0 ) out << '\"' << units_att << '\"'; else out << "(nul)"; +out << prefix << "ValidTime = " << unix_to_yyyymmdd_hhmmss(ValidTime) + << " (" << ValidTime << ")\n"; + +out << prefix << "InitTime = " << unix_to_yyyymmdd_hhmmss(InitTime) + << " (" << InitTime << ")\n"; + out << prefix << "AccumTime = " << AccumTime; out << "\n"; @@ -232,6 +242,17 @@ return; } +//////////////////////////////////////////////////////////////////////// + + +int NcVarInfo::lead_time() const + +{ + +return ( (int) (ValidTime - InitTime) ); + +} + //////////////////////////////////////////////////////////////////////// @@ -254,6 +275,10 @@ level_att = i.level_att; units_att = i.units_att; +ValidTime = i.ValidTime; + +InitTime = i.InitTime; + AccumTime = i.AccumTime; Ndims = i.Ndims; diff --git a/met/src/libcode/vx_nc_util/nc_var_info.h b/met/src/libcode/vx_nc_util/nc_var_info.h index ba86dcd011..594677734b 100644 --- a/met/src/libcode/vx_nc_util/nc_var_info.h +++ b/met/src/libcode/vx_nc_util/nc_var_info.h @@ -63,8 +63,14 @@ class NcVarInfo { ConcatString units_att; + unixtime ValidTime; + + unixtime InitTime; + int AccumTime; // seconds + int lead_time () const; // seconds + int Ndims; NcDim ** Dims; // allocated diff --git a/test/config/GridStatConfig_fourier b/test/config/GridStatConfig_fourier index ad8c121b23..ae4e33c118 100644 --- a/test/config/GridStatConfig_fourier +++ b/test/config/GridStatConfig_fourier @@ -47,7 +47,7 @@ mpr_thresh = []; cat_thresh = []; cnt_thresh = [ NA ]; cnt_logic = UNION; -wind_thresh = [ NA ]; +wind_thresh = [ >5.0 ]; wind_logic = UNION; eclv_points = 0.05; nc_pairs_var_name = "";