From edc669e0694bd266f915e4dbdee027df58c425a6 Mon Sep 17 00:00:00 2001 From: johnhg Date: Thu, 21 Apr 2022 13:32:05 -0600 Subject: [PATCH 01/10] Update main_v10.1-ref after #2144 (#2145) Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Howard Soh Co-authored-by: hsoh-u --- .github/dummy_for_action | 2 - .github/jobs/set_job_controls.sh | 17 +++ met/docs/Users_Guide/release-notes.rst | 9 ++ met/docs/conf.py | 4 +- met/src/basic/vx_util/data_plane_util.cc | 9 +- met/src/basic/vx_util/data_plane_util.h | 2 +- met/src/basic/vx_util/util_constants.h | 3 +- met/src/libcode/vx_data2d_grib/data2d_grib.cc | 4 + .../libcode/vx_data2d_grib/grib_classes.cc | 21 ++++ met/src/libcode/vx_data2d_grib/grib_classes.h | 30 ++++- met/src/libcode/vx_data2d_grib/grib_utils.cc | 113 +++++++++++++++--- .../libcode/vx_data2d_grib2/data2d_grib2.cc | 2 +- .../libcode/vx_data2d_nc_met/get_met_grid.cc | 2 +- met/src/libcode/vx_data2d_nccf/nccf_file.cc | 21 ++-- met/src/libcode/vx_nc_util/nc_utils.cc | 57 +++++++-- met/src/tools/other/ioda2nc/ioda2nc.cc | 4 +- .../tools/other/plot_point_obs/Makefile.am | 2 +- scripts/environment/development.docker | 44 +++++++ 18 files changed, 292 insertions(+), 54 deletions(-) delete mode 100644 .github/dummy_for_action create mode 100644 scripts/environment/development.docker diff --git a/.github/dummy_for_action b/.github/dummy_for_action deleted file mode 100644 index 91bdf4d63e..0000000000 --- a/.github/dummy_for_action +++ /dev/null @@ -1,2 +0,0 @@ -#update me to add action comment - diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh index 6e640930d2..03d8a2fe32 100755 --- a/.github/jobs/set_job_controls.sh +++ b/.github/jobs/set_job_controls.sh @@ -17,6 +17,13 @@ if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then run_diff=true + # if pull request into main_vX.Y branch, + # set truth data version to branch name and set input data version to X.Y + if [ "${GITHUB_BASE_REF:0:6}" == "main_v" ]; then + truth_data_version=${GITHUB_BASE_REF} + input_data_version=${GITHUB_BASE_REF:6} + fi + fi elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then @@ -30,6 +37,11 @@ elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then run_diff=true truth_data_version=${branch_name:0: -4} + # if main_vX.Y-ref branch, use X.Y input data + if [ "${branch_name:0:6}" == "main_v" ]; then + input_data_version=${branch_name:6: -4} + fi + else # if develop or main_vX.Y branch, run diff tests using branch's truth data @@ -39,6 +51,11 @@ elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then run_diff=true truth_data_version=${branch_name} + # if main_vX.Y branch, use X.Y input data + if [ "${branch_name:0:6}" == "main_v" ]; then + input_data_version=${branch_name:6} + fi + fi # check commit messages for skip or force keywords diff --git a/met/docs/Users_Guide/release-notes.rst b/met/docs/Users_Guide/release-notes.rst index d5fa2e5ec7..9252713b88 100644 --- a/met/docs/Users_Guide/release-notes.rst +++ b/met/docs/Users_Guide/release-notes.rst @@ -5,6 +5,15 @@ When applicable, release notes are followed by the GitHub issue number which des enhancement, or new feature (`MET GitHub issues `_). Important issues are listed **in bold** for emphasis. +MET Version 10.1.1 release notes (20220419) +------------------------------------------- + +* Bugfixes: + + * Fix support for reading rotated lat/lon grids from CF-compliant NetCDF files (`#2115 `_). + * Fix support for reading rotated lat/lon grids from GRIB1 files (grid type 10) (`#2118 `_). + * Fix support for int64 NetCDF variable types (`#2123 `_). + MET Version 10.1.0 release notes (20220314) ------------------------------------------- diff --git a/met/docs/conf.py b/met/docs/conf.py index 44bbb2f91f..ae0069a127 100644 --- a/met/docs/conf.py +++ b/met/docs/conf.py @@ -20,11 +20,11 @@ project = 'MET' author = 'UCAR/NCAR, NOAA, CSU/CIRA, and CU/CIRES' author_list = 'Newman, K., J. Opatz, T. Jensen, J. Prestopnik, H. Soh, L. Goodrich, B. Brown, R. Bullock, J. Halley Gotway' -version = '10.1.0' +version = '10.1.1' verinfo = version release = f'{version}' release_year = '2022' -release_date = f'{release_year}-03-14' +release_date = f'{release_year}-04-19' copyright = f'{release_year}, {author}' # -- General configuration --------------------------------------------------- diff --git a/met/src/basic/vx_util/data_plane_util.cc b/met/src/basic/vx_util/data_plane_util.cc index 8f5feea095..d34e56a361 100644 --- a/met/src/basic/vx_util/data_plane_util.cc +++ b/met/src/basic/vx_util/data_plane_util.cc @@ -30,7 +30,6 @@ using namespace std; #include "GridTemplate.h" - //////////////////////////////////////////////////////////////////////// // // Utility functions operating on a DataPlane @@ -206,7 +205,7 @@ DataPlane smooth_field(const DataPlane &dp, //////////////////////////////////////////////////////////////////////// void fractional_coverage(const DataPlane &dp, DataPlane &frac_dp, - int width, const GridTemplateFactory::GridTemplates shape, + int width, GridTemplateFactory::GridTemplates shape, bool wrap_lon, SingleThresh t, const DataPlane *cmn, const DataPlane *csd, double vld_t) { GridPoint *gp = NULL; @@ -251,9 +250,9 @@ void fractional_coverage(const DataPlane &dp, DataPlane &frac_dp, } } -#pragma omp parallel default(none) \ - shared(mlog, dp, frac_dp, width, wrap_lon, t) \ - shared(use_climo, cmn, csd, vld_t, bad) \ +#pragma omp parallel default(none) \ + shared(mlog, dp, frac_dp, shape, width, wrap_lon, t) \ + shared(use_climo, cmn, csd, vld_t, bad) \ private(x, y, n_vld, n_thr, gp, v) { diff --git a/met/src/basic/vx_util/data_plane_util.h b/met/src/basic/vx_util/data_plane_util.h index 1b5788dddc..c4f0b07c16 100644 --- a/met/src/basic/vx_util/data_plane_util.h +++ b/met/src/basic/vx_util/data_plane_util.h @@ -51,7 +51,7 @@ extern DataPlane smooth_field(const DataPlane &dp, bool wrap_lon, double t, const GaussianInfo &gaussian); extern void fractional_coverage(const DataPlane &dp, DataPlane &frac_dp, - int width, const GridTemplateFactory::GridTemplates shape, + int width, GridTemplateFactory::GridTemplates shape, bool wrap_lon, SingleThresh t, const DataPlane *cmn, const DataPlane *csd, double vld_t); diff --git a/met/src/basic/vx_util/util_constants.h b/met/src/basic/vx_util/util_constants.h index 34da0d9e91..4a40628fa3 100644 --- a/met/src/basic/vx_util/util_constants.h +++ b/met/src/basic/vx_util/util_constants.h @@ -18,6 +18,7 @@ //////////////////////////////////////////////////////////////////////// // Released versions of MET +static const char met_version_10_1_1[] = "V10.1.1"; static const char met_version_10_1_0[] = "V10.1.0"; static const char met_version_10_0_0[] = "V10.0.0"; static const char met_version_9_1[] = "V9.1"; @@ -40,7 +41,7 @@ static const char met_version_1_1[] = "V1.1"; //////////////////////////////////////////////////////////////////////// -static const char * const met_version = met_version_10_1_0; +static const char * const met_version = met_version_10_1_1; static const char default_met_data_dir[] = "MET_BASE"; static const char txt_file_ext[] = ".txt"; static const char stat_file_ext[] = ".stat"; diff --git a/met/src/libcode/vx_data2d_grib/data2d_grib.cc b/met/src/libcode/vx_data2d_grib/data2d_grib.cc index ef5e622b9c..c92e64569a 100644 --- a/met/src/libcode/vx_data2d_grib/data2d_grib.cc +++ b/met/src/libcode/vx_data2d_grib/data2d_grib.cc @@ -762,6 +762,10 @@ bool is_grid_relative(const GribRecord &r) { else if(r.gds->type == 5) { res_flag = r.gds->grid_type.stereographic.res_flag; } + // Rotated LatLon + else if(r.gds->type == 10) { + res_flag = r.gds->grid_type.rot_latlon_grid.res_flag; + } else { mlog << Error << "\nis_grid_relative() -> " << "Unsupported grid type value: " << r.gds->type diff --git a/met/src/libcode/vx_data2d_grib/grib_classes.cc b/met/src/libcode/vx_data2d_grib/grib_classes.cc index da1a5f0f76..d02445c6ef 100644 --- a/met/src/libcode/vx_data2d_grib/grib_classes.cc +++ b/met/src/libcode/vx_data2d_grib/grib_classes.cc @@ -1874,6 +1874,27 @@ else if ((h.type == 5)) file << " scan_flag: " << (int) h.grid_type.stereographic.scan_flag << "\n\n"; } +else if ((h.type == 10)) +{ + + file << " lat1: " << char3_to_int(h.grid_type.rot_latlon_grid.lat1) << "\n"; + file << " lon1: " << char3_to_int(h.grid_type.rot_latlon_grid.lon1) << "\n"; + + file << " res_flag: " << (int) h.grid_type.rot_latlon_grid.res_flag << "\n"; + + file << " lat2: " << char3_to_int(h.grid_type.rot_latlon_grid.lat2) << "\n"; + file << " lon2: " << char3_to_int(h.grid_type.rot_latlon_grid.lon2) << "\n"; + + file << " di: " << char2_to_int(h.grid_type.rot_latlon_grid.di) << "\n"; + file << " dj: " << char2_to_int(h.grid_type.rot_latlon_grid.dj) << "\n"; + + file << " scan_flag: " << (int) h.grid_type.rot_latlon_grid.scan_flag << "\n"; + + file << " lat_sp: " << char3_to_int(h.grid_type.rot_latlon_grid.lat_sp) << "\n"; + file << " lon_sp: " << char3_to_int(h.grid_type.rot_latlon_grid.lon_sp) << "\n"; + + file << " rotation: " << char4_to_dbl(h.grid_type.rot_latlon_grid.rotation) << "\n\n"; +} return ( file ); diff --git a/met/src/libcode/vx_data2d_grib/grib_classes.h b/met/src/libcode/vx_data2d_grib/grib_classes.h index 55b7046d65..3b0295cf45 100644 --- a/met/src/libcode/vx_data2d_grib/grib_classes.h +++ b/met/src/libcode/vx_data2d_grib/grib_classes.h @@ -162,7 +162,7 @@ struct Section1_Header { // PDS //////////////////////////////////////////////////////////////////////// -struct LatLon { // Latitude/Longitude Grid +struct LatLon { // Latitude/Longitude Grid unsigned char lat1[3]; // 11 - 13 unsigned char lon1[3]; // 14 - 16 @@ -179,6 +179,32 @@ struct LatLon { // Latitude/Longitude Grid unsigned char unused[14]; // 29 - 42 +}; + + // Reference: https://apps.ecmwf.int/codes/grib/format/grib1/grids/10 + +struct RotLatLon { // Rotated Latitude/Longitude Grid + + unsigned char lat1[3]; // 11 - 13 + unsigned char lon1[3]; // 14 - 16 + + unsigned char res_flag; // 17 + + unsigned char lat2[3]; // 18 - 20 + unsigned char lon2[3]; // 21 - 23 + + unsigned char di[2]; // 24 - 25 + unsigned char dj[2]; // 26 - 27 + + unsigned char scan_flag; // 28 + + unsigned char unused[4]; // 29 - 32 + + unsigned char lat_sp[3]; // 33 - 35 + unsigned char lon_sp[3]; // 36 - 38 + + unsigned char rotation[4]; // 39 - 42 + }; struct Mercator { // Mercator Grid @@ -282,7 +308,7 @@ struct Gaussian { union GridType { struct LatLon latlon_grid; // Latitude/Longitude Grid - // struct RotLatLon rot_latlon_grid; // Rotated Latitude/Longitude Grid + struct RotLatLon rot_latlon_grid; // Rotated Latitude/Longitude Grid struct Mercator mercator; // Mercator Grid struct LambertConf lambert_conf; // Lambert Conformal Secant Grid struct Stereographic stereographic; // Stereographic Grid diff --git a/met/src/libcode/vx_data2d_grib/grib_utils.cc b/met/src/libcode/vx_data2d_grib/grib_utils.cc index 6dc5b7c563..c1ef70f775 100644 --- a/met/src/libcode/vx_data2d_grib/grib_utils.cc +++ b/met/src/libcode/vx_data2d_grib/grib_utils.cc @@ -29,11 +29,13 @@ using namespace std; // grid types from the GDS section // -static const int latlon_type = 0; -static const int mercator_type = 1; -static const int lambert_type = 3; -static const int stereographic_type = 5; -static const int gaussian_type = 4; +static const int latlon_type = 0; +static const int mercator_type = 1; +static const int lambert_type = 3; +static const int gaussian_type = 4; +static const int stereographic_type = 5; +static const int rotated_latlon_type = 10; + //////////////////////////////////////////////////////////////////////// @@ -62,7 +64,7 @@ void gds_to_grid(const Section2_Header & gds, Grid & gr) LambertData lc_data; StereographicData st_data; LatLonData ll_data; -//RotatedLatLonData rll_data; +RotatedLatLonData rll_data; MercatorData mc_data; GaussianData g_data; @@ -84,11 +86,11 @@ if ( gds.type == latlon_type ) { gr.set(ll_data); -// else if ( gds.type == rotated_latlon_type ) { -// -// gds_to_rotated_latlon(gds, rll_data); -// -// gr.set(rll_data); +} else if ( gds.type == rotated_latlon_type ) { + + gds_to_rotated_latlon(gds, rll_data); + + gr.set(rll_data); } else if ( gds.type == mercator_type ) { @@ -143,6 +145,7 @@ void gds_to_order(const Section2_Header & gds, int & xdir, int & ydir, int & ord // Check GDS for the grid type. // The following Projection types are supported: // - Lat/Lon + // - Rotated Lat/Lon // - Mercator // - Lambert Conformal // - Polar Stereographic @@ -154,6 +157,10 @@ if ( gds.type == latlon_type ) { scan_flag_to_order(gds.grid_type.latlon_grid.scan_flag, xdir, ydir, order); +} else if ( gds.type == rotated_latlon_type ) { + + scan_flag_to_order(gds.grid_type.rot_latlon_grid.scan_flag, xdir, ydir, order); + } else if (gds.type == mercator_type ) { scan_flag_to_order(gds.grid_type.mercator.scan_flag, xdir, ydir, order); @@ -278,10 +285,88 @@ void gds_to_rotated_latlon(const Section2_Header & gds, RotatedLatLonData & data { -mlog << Error << "\ngds_to_rotated_latlon() -> " - << "Rotated Lat/Lon grids are not supported in GRIB version 1.\n\n"; +int xdir, ydir, order; +double d; + +scan_flag_to_order(gds.grid_type.rot_latlon_grid.scan_flag, xdir, ydir, order); + + // Store the grid name +data.name = rotated_latlon_proj_type; + + // + // Multiply longitude values by -1 since the NCAR code considers + // degrees west to be positive. + // + + // Latitude of the bottom left corner +data.rot_lat_ll = min(decode_lat_lon(gds.grid_type.rot_latlon_grid.lat1, 3), + decode_lat_lon(gds.grid_type.rot_latlon_grid.lat2, 3)); + + // Longitude of the bottom left corner +if ( xdir == 0 ) data.rot_lon_ll = -1.0*rescale_lon(decode_lat_lon(gds.grid_type.rot_latlon_grid.lon1, 3)); +else data.rot_lon_ll = -1.0*rescale_lon(decode_lat_lon(gds.grid_type.rot_latlon_grid.lon2, 3)); + + // Number of points in the Latitudinal (y) direction +data.Nlat = char2_to_int(gds.ny); + + // Number of points in the Longitudinal (x) direction +data.Nlon = char2_to_int(gds.nx); + + // Check for thinned lat/lon grids +if ( data.Nlon == 65535 ) { -exit ( 1 ); + mlog << Error << "\ngds_to_rotated_latlon() -> " + << "Thinned Lat/Lon grids are not supported for GRIB version 1.\n\n"; + + exit ( 1 ); + +} + + // Compute latitudinal increment from lat1 and lat2 +d = fabs(decode_lat_lon(gds.grid_type.rot_latlon_grid.lat1, 3) + - decode_lat_lon(gds.grid_type.rot_latlon_grid.lat2, 3)); + +data.delta_rot_lat = d/(data.Nlat); + + // If explicitly specified and non-zero, use it +if ( ! all_bits_set(gds.grid_type.rot_latlon_grid.dj, 2) ) { + + d = decode_lat_lon(gds.grid_type.rot_latlon_grid.dj, 2); + + if ( d > 0 ) data.delta_rot_lat = d; + +} + + // Compute longitudinal increment from lon1 and lon2 +d = fabs(decode_lat_lon(gds.grid_type.rot_latlon_grid.lon1, 3) + - decode_lat_lon(gds.grid_type.rot_latlon_grid.lon2, 3)); + +data.delta_rot_lon = d/(data.Nlon); + + // If explicitly specified and non-zero, use it +if ( ! all_bits_set(gds.grid_type.rot_latlon_grid.di, 2) ) { + + d = decode_lat_lon(gds.grid_type.rot_latlon_grid.di, 2); + + if ( d > 0 ) data.delta_rot_lon = d; + +} + + // Location of (rotated) south pole +data.true_lat_south_pole = + decode_lat_lon(gds.grid_type.rot_latlon_grid.lat_sp, 3); + +data.true_lon_south_pole = + -1.0*rescale_lon(decode_lat_lon(gds.grid_type.rot_latlon_grid.lon_sp, 3)); + + // Auxiliary rotation +data.aux_rotation = char4_to_dbl(gds.grid_type.rot_latlon_grid.rotation); + +data.dump(); + + // + // done + // return; diff --git a/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc b/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc index 0061d2fd00..65df4c7ef7 100644 --- a/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc +++ b/met/src/libcode/vx_data2d_grib2/data2d_grib2.cc @@ -1100,7 +1100,7 @@ void MetGrib2DataFile::read_grib2_grid( gribfield *gfld) { data.true_lon_south_pole = s_lon; // - // auxilliary rotation around the rotated polar axis + // auxiliary rotation around the rotated polar axis // data.aux_rotation = (t[21])*angle_factor; diff --git a/met/src/libcode/vx_data2d_nc_met/get_met_grid.cc b/met/src/libcode/vx_data2d_nc_met/get_met_grid.cc index 8e9e68cf4d..3512c6aaed 100644 --- a/met/src/libcode/vx_data2d_nc_met/get_met_grid.cc +++ b/met/src/libcode/vx_data2d_nc_met/get_met_grid.cc @@ -309,7 +309,7 @@ data.rot_lon_ll *= -1.0; get_global_att(ncfile, string("true_lon_south_pole"), data.true_lon_south_pole); if ( !west_longitude_positive ) data.true_lon_south_pole *= -1.0; - // auxilliary rotation + // auxiliary rotation get_global_att(ncfile, string("aux_rotation"), data.aux_rotation); diff --git a/met/src/libcode/vx_data2d_nccf/nccf_file.cc b/met/src/libcode/vx_data2d_nccf/nccf_file.cc index 7ccdd710ae..7dcb7dc99a 100644 --- a/met/src/libcode/vx_data2d_nccf/nccf_file.cc +++ b/met/src/libcode/vx_data2d_nccf/nccf_file.cc @@ -296,17 +296,15 @@ bool NcCfFile::open(const char * filepath) } // Parse the units for the time variable. + ut = sec_per_unit = 0; NcVarAtt *units_att = get_nc_att(valid_time_var, (string)"units", false); if (IS_VALID_NC_P(units_att)) { - get_att_value_chars(units_att, units); - - if (units.length() == 0) + if (!get_att_value_chars(units_att, units) || units.length() == 0) { mlog << Warning << "\n" << method_name << "the \"time\" variable must contain a \"units\" attribute. " << "Using valid time of 0\n\n"; - ut = sec_per_unit = 0; } else { @@ -315,14 +313,10 @@ bool NcCfFile::open(const char * filepath) parse_cf_time_string(units.c_str(), ut, sec_per_unit); } } - else - { - ut = sec_per_unit = 0; - } + if (units_att) delete units_att; NcVar bounds_time_var; NcVar *nc_time_var = (NcVar *)0; - nc_time_var = valid_time_var; bool use_bounds_var = false; ConcatString bounds_var_name; nc_time_var = valid_time_var; @@ -338,12 +332,12 @@ bool NcCfFile::open(const char * filepath) } if (bounds_att) delete bounds_att; - if (units_att) delete units_att; // Determine the number of times present. int n_times = (int) get_data_size(valid_time_var); int tim_buf_size = n_times; if (use_bounds_var) tim_buf_size *= 2; double *time_values = new double[tim_buf_size]; + if( get_nc_data(nc_time_var, time_values) ) { bool no_leap_year = get_att_no_leap_year(valid_time_var); if( time_dim_count > 1 ) { @@ -375,6 +369,7 @@ bool NcCfFile::open(const char * filepath) } } } + else ValidTime.add(0); //Initialize delete [] time_values; } @@ -2430,10 +2425,12 @@ void NcCfFile::get_grid_mapping_rotated_latitude_longitude(const NcVar *grid_map data.name = rotated_latlon_proj_type; - // Derive south pole location from the north pole + // Derive south pole location from the north pole: + // - Reverse the sign of the latitude + // - Add 180 to the longitude and switch from degrees east to west data.true_lat_south_pole = -1.0 * get_att_value_double(grid_np_lat_att); double np_lon = rescale_lon(get_att_value_double(grid_np_lon_att)); - data.true_lon_south_pole = rescale_lon(-1.0 * (180.0 - fabs(np_lon))); + data.true_lon_south_pole = rescale_lon(-1.0 * (np_lon + 180.0)); // Copied from the LatLon data structure data.rot_lat_ll = ll_data.lat_ll; diff --git a/met/src/libcode/vx_nc_util/nc_utils.cc b/met/src/libcode/vx_nc_util/nc_utils.cc index 8d8e5b1087..0fa9f87df9 100644 --- a/met/src/libcode/vx_nc_util/nc_utils.cc +++ b/met/src/libcode/vx_nc_util/nc_utils.cc @@ -1533,6 +1533,7 @@ bool get_nc_data(NcVar *var, float *data) { // Note: missing data was checked here // int type_id = GET_NC_TYPE_ID_P(var); + return_status = true; if (NcType::nc_FLOAT == type_id) { var->getVar(data); } @@ -1565,9 +1566,26 @@ bool get_nc_data(NcVar *var, float *data) { if (IS_VALID_NC_P(att_scale_factor)) scale_factor = get_att_value_float(att_scale_factor); mlog << Debug(4) << method_name << "add_offset = " << add_offset << ", scale_factor=" << scale_factor << ", cell_count=" << cell_count - << ", is_unsigned_value: " << unsigned_value << "\n"; + << ", is_unsigned_value: " << unsigned_value << " for " << GET_NC_NAME_P(var) << "\n"; switch ( type_id ) { + case NcType::nc_INT64: + { + long long fill_value = bad_data_int; + long long min_value = 2147483647; + long long max_value = -2147483648; + long long *packed_data = new long long[cell_count]; + + if (IS_VALID_NC_P(att_fill_value)) + fill_value = get_att_value_int(att_fill_value); + + var->getVar(packed_data); + _apply_scale_factor(data, packed_data, cell_count, + fill_value, min_value, max_value, "int64", + add_offset, scale_factor); + delete [] packed_data; + } + break; case NcType::nc_INT: { int fill_value = bad_data_int; @@ -1720,15 +1738,15 @@ bool get_nc_data(NcVar *var, float *data) { } break; default: - mlog << Debug(1) << method_name << "type_id: " - << type_id << " type name: " << GET_NC_TYPE_NAME_P(var) - << "\n"; + return_status = false; + mlog << Debug(1) << method_name << "Did not read data because of unsupported data type (" + << type_id << ", type name: " << GET_NC_TYPE_NAME_P(var) + << ") for " << GET_NC_NAME_P(var) << "\n"; } if(att_add_offset) delete att_add_offset; if(att_scale_factor) delete att_scale_factor; if(att_fill_value) delete att_fill_value; } - return_status = true; } mlog << Debug(6) << method_name << "took " @@ -1821,6 +1839,7 @@ bool get_nc_data(NcVar *var, double *data) { // int unpacked_count = 0; int type_id = GET_NC_TYPE_ID_P(var); + return_status = true; if ((NcType::nc_DOUBLE == type_id) || (NcType::nc_FLOAT == type_id)){ var->getVar(data); } @@ -1844,9 +1863,26 @@ bool get_nc_data(NcVar *var, double *data) { } mlog << Debug(4) << method_name << "add_offset = " << add_offset << ", scale_factor=" << scale_factor << ", cell_count=" << cell_count - << ", is_unsigned_value: " << unsigned_value << "\n"; + << ", is_unsigned_value: " << unsigned_value << " for " << GET_NC_NAME_P(var) << "\n"; switch ( type_id ) { + case NcType::nc_INT64: + { + long long fill_value = bad_data_int; + long long min_value = 2147483647; + long long max_value = -2147483648; + long long *packed_data = new long long[cell_count]; + + if (IS_VALID_NC_P(att_fill_value)) + fill_value = get_att_value_int(att_fill_value); + + var->getVar(packed_data); + _apply_scale_factor(data, packed_data, cell_count, + fill_value, min_value, max_value, "int64", + add_offset, scale_factor); + delete [] packed_data; + } + break; case NcType::nc_INT: { int fill_value = bad_data_int; @@ -2001,15 +2037,16 @@ bool get_nc_data(NcVar *var, double *data) { } break; default: - mlog << Debug(1) << method_name << "type_id: " - << type_id << " type name: " << GET_NC_TYPE_NAME_P(var) - << "\n"; + return_status = false; + mlog << Debug(1) << method_name << "Did not read data because of unsupported data type (" + << type_id << ", type name: " << GET_NC_TYPE_NAME_P(var) + << ") for " << GET_NC_NAME_P(var) << "\n"; + } if(att_add_offset) delete att_add_offset; if(att_scale_factor) delete att_scale_factor; if(att_fill_value) delete att_fill_value; } - return_status = true; } return(return_status); } diff --git a/met/src/tools/other/ioda2nc/ioda2nc.cc b/met/src/tools/other/ioda2nc/ioda2nc.cc index cc36eed5fe..00244d20fb 100644 --- a/met/src/tools/other/ioda2nc/ioda2nc.cc +++ b/met/src/tools/other/ioda2nc/ioda2nc.cc @@ -379,7 +379,7 @@ void process_ioda_file(int i_pb) { bool apply_poly_mask = (conf_info.poly_mask.n_points() > 0); hdr_typ[0] = 0; - + file_ut = beg_ut = end_ut = hdr_vld_ut = (unixtime) 0; // List the IODA file being processed @@ -403,11 +403,11 @@ void process_ioda_file(int i_pb) { min_msg_ut = max_msg_ut = (unixtime) 0; min_time_str[0] = 0; max_time_str[0] = 0; + modified_hdr_typ[0] = 0; // Set the file name for the IODA file file_name << ioda_files[i_pb]; - int nrecs = 0; //int nstring = 0; StringArray var_names, dim_names; diff --git a/met/src/tools/other/plot_point_obs/Makefile.am b/met/src/tools/other/plot_point_obs/Makefile.am index fffb055719..690bb9ab63 100644 --- a/met/src/tools/other/plot_point_obs/Makefile.am +++ b/met/src/tools/other/plot_point_obs/Makefile.am @@ -44,7 +44,7 @@ plot_point_obs_LDADD = -lvx_statistics \ -lvx_cal \ -lvx_log \ $(PYTHON_LIBS) \ - -lm -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas + -lm -lz -lnetcdf_c++4 -lnetcdf -lgsl -lgslcblas EXTRA_DIST = plot_point_obs.h \ plot_point_obs_conf_info.h diff --git a/scripts/environment/development.docker b/scripts/environment/development.docker new file mode 100644 index 0000000000..ec4bd202c3 --- /dev/null +++ b/scripts/environment/development.docker @@ -0,0 +1,44 @@ +# Define the development environment for NCAR project machine seneca +# Based on settings in /usr/local/src/met/README.snat + +# Top-level MET project directory +MET_PROJ_DIR=`ls -1d /met/MET*` + +# Variables required to build MET +export MET_DEVELOPMENT=true + +export MET_NETCDF=/usr/local +export MET_HDF5INC=/usr/include +export MET_HDF5LIB=/usr/lib64 +export MET_HDF=/usr/local/hdf +export MET_HDFEOS=/usr/local/hdfeos +export MET_BUFR=/usr/local +export MET_GRIB2CINC=/usr/include +export MET_GRIB2CLIB=/usr/lib64 +export MET_GSLINC=/usr/include/gsl +export MET_GSLLIB=/usr/lib64 +export MET_CAIROINC=/usr/include/cairo +export MET_CAIROLIB=/usr/lib64 +export MET_FREETYPEINC=/usr/include/freetype2 +export MET_FREETYPELIB=/usr/lib64 +export MET_JASPERLIB=/usr/lib64 + +export MET_PYTHON=/usr/bin/python3 +export MET_PYTHON_CC="-I/usr/include/python3.6m -I/usr/include/python3.6m" +export MET_PYTHON_LD="-L/usr/lib64 -lpython3.6m -lpthread -ldl -lutil -lm" + +# -D__64BIT__ is required because we've compiled libgrib2c.a with that flag +export CFLAGS="-DUNDERSCORE -fPIC -D__64BIT__ -g" +export CXXFLAGS=${CFLAGS} + +# Set LDFLAGS to include -rpath settings when compiling MET +export LDFLAGS="-Wl,--disable-new-dtags" +export LDFLAGS="${LDFLAGS} -Wl,-rpath,/usr/local/lib:/usr/lib64:${MET_HDF}/lib:${MET_HDFEOS}/lib" + +# Variables required to run MET +export MET_TEST_INPUT=${MET_PROJ_DIR}/MET_test_data/unit_test +export MET_FONT_DIR=${MET_TEST_INPUT}/fonts + +# Define Rscript to use a version with the ncdf4 package 1.17 or later +export MET_TEST_RSCRIPT=/usr/bin/Rscript + From 6b6e6404b226c6262a84fa45ab52b7c5f05c4171 Mon Sep 17 00:00:00 2001 From: johnhg Date: Thu, 5 May 2022 16:25:03 -0600 Subject: [PATCH 02/10] Add testing and docs badges. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ee97ac3082..bb0b829264 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Model Evaluation Tools (MET) Repository ======================================= +[![Tests](https://github.com/DTCenter/MET/actions/workflows/testing.yml/badge.svg?event=push)](https://github.com/DTCenter/MET/actions/workflows/testing.yml) +[![Docs](https://img.shields.io/badge/Documentation-latest-brightgreen.svg)](https://met.readthedocs.io) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5565322.svg)](https://doi.org/10.5281/zenodo.5565322) This repository contains the source code for the Model Evaluation Tools package (met), unit test code (test), and scripts used to build and test the code (scripts). From 0a5664bb3345c132e3decda832da84b5f975862a Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 9 May 2022 22:01:52 -0600 Subject: [PATCH 03/10] #14 Using literal instead of numbers and make sure no overflow --- met/src/basic/vx_log/concat_string.cc | 12 +++++++++--- met/src/basic/vx_log/concat_string.h | 3 ++- met/src/basic/vx_util/ascii_table.cc | 20 ++++++++++++++------ met/src/basic/vx_util/ascii_table.h | 5 +++-- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/met/src/basic/vx_log/concat_string.cc b/met/src/basic/vx_log/concat_string.cc index 36ea8b17a1..03669e9e28 100644 --- a/met/src/basic/vx_log/concat_string.cc +++ b/met/src/basic/vx_log/concat_string.cc @@ -231,7 +231,10 @@ void ConcatString::assign(const ConcatString & c) if (c.text()) s->assign(c.text()); else s->clear(); - memcpy(FloatFormat, c.FloatFormat, sizeof(FloatFormat)); + int buf_size = sizeof(c.FloatFormat); + if (buf_size > concat_string_buf_size) buf_size = concat_string_buf_size; + + memcpy(FloatFormat, c.FloatFormat, buf_size); Precision = c.Precision; } @@ -337,9 +340,12 @@ if ( (k < 0) || (k > concat_string_max_precision) ) { if (Precision != k) { Precision = k; - memset(FloatFormat, 0, sizeof(FloatFormat)); + int buf_size = sizeof(FloatFormat); + if (buf_size > concat_string_buf_size) buf_size = concat_string_buf_size; + + memset(FloatFormat, 0, buf_size); - snprintf(FloatFormat, sizeof(FloatFormat), "%%.%df", Precision); + snprintf(FloatFormat, buf_size, "%%.%df", Precision); } return; diff --git a/met/src/basic/vx_log/concat_string.h b/met/src/basic/vx_log/concat_string.h index df41cbf78e..59dec5f4f6 100644 --- a/met/src/basic/vx_log/concat_string.h +++ b/met/src/basic/vx_log/concat_string.h @@ -47,6 +47,7 @@ static const int max_str_len = 512; static const int concat_string_default_precision = 5; static const int concat_string_max_precision = 12; +static const int concat_string_buf_size = concat_string_max_precision + 4; //////////////////////////////////////////////////////////////////////// @@ -73,7 +74,7 @@ class ConcatString { int Precision; - char FloatFormat[16]; + char FloatFormat[concat_string_buf_size]; std::string *s; diff --git a/met/src/basic/vx_util/ascii_table.cc b/met/src/basic/vx_util/ascii_table.cc index fccfede13b..343a5aad23 100644 --- a/met/src/basic/vx_util/ascii_table.cc +++ b/met/src/basic/vx_util/ascii_table.cc @@ -271,8 +271,12 @@ BadDataValue = a.BadDataValue; set_bad_data_str(a.BadDataStr); -memcpy(f_FloatFormat, a.f_FloatFormat, sizeof(f_FloatFormat)); -memcpy(g_FloatFormat, a.g_FloatFormat, sizeof(g_FloatFormat)); +int f_buf_size = sizeof(a.f_FloatFormat); +int g_buf_size = sizeof(a.g_FloatFormat); +if (f_buf_size > ascii_table_buf_size) f_buf_size = ascii_table_buf_size; +if (g_buf_size > ascii_table_buf_size) g_buf_size = ascii_table_buf_size; +memcpy(f_FloatFormat, a.f_FloatFormat, f_buf_size); +memcpy(g_FloatFormat, a.g_FloatFormat, g_buf_size); DoCommaString = a.DoCommaString; @@ -697,11 +701,15 @@ if ( (k < 0) || (k > ascii_table_max_precision) ) { Precision = k; -memset(f_FloatFormat, 0, sizeof(f_FloatFormat)); -memset(g_FloatFormat, 0, sizeof(g_FloatFormat)); +int f_buf_size = sizeof(f_FloatFormat); +int g_buf_size = sizeof(g_FloatFormat); +if (f_buf_size > ascii_table_buf_size) f_buf_size = ascii_table_buf_size; +if (g_buf_size > ascii_table_buf_size) g_buf_size = ascii_table_buf_size; +memset(f_FloatFormat, 0, f_buf_size); +memset(g_FloatFormat, 0, g_buf_size); -snprintf(f_FloatFormat, sizeof(f_FloatFormat), "%%.%df", Precision); -snprintf(g_FloatFormat, sizeof(g_FloatFormat), "%%.%dg", Precision); +snprintf(f_FloatFormat, f_buf_size, "%%.%df", Precision); +snprintf(g_FloatFormat, g_buf_size, "%%.%dg", Precision); return; diff --git a/met/src/basic/vx_util/ascii_table.h b/met/src/basic/vx_util/ascii_table.h index a0f32056fd..c13c67c904 100644 --- a/met/src/basic/vx_util/ascii_table.h +++ b/met/src/basic/vx_util/ascii_table.h @@ -112,6 +112,7 @@ static const bool default_fill_blank = false; static const int ascii_table_default_precision = 2; static const int ascii_table_max_precision = 12; +static const int ascii_table_buf_size = ascii_table_max_precision + 4; static const double ascii_table_default_bad_data_value = -9999.0; @@ -163,8 +164,8 @@ class AsciiTable { std::string BadDataStr; - char f_FloatFormat[16]; - char g_FloatFormat[16]; + char f_FloatFormat[ascii_table_buf_size]; + char g_FloatFormat[ascii_table_buf_size]; bool DoCommaString; // do comma string? From f0579fcb77588bd49a2aa91088a37f40356faba2 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 9 May 2022 22:03:49 -0600 Subject: [PATCH 04/10] #14 Using literal instead of numbers --- met/src/tools/other/gsi_tools/rad_record.cc | 46 ++++++++++----------- met/src/tools/other/gsi_tools/rad_record.h | 13 ++++-- 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/met/src/tools/other/gsi_tools/rad_record.cc b/met/src/tools/other/gsi_tools/rad_record.cc index 4ce91f1202..a87b453853 100644 --- a/met/src/tools/other/gsi_tools/rad_record.cc +++ b/met/src/tools/other/gsi_tools/rad_record.cc @@ -404,23 +404,23 @@ if ( n_read < 72 ) { unsigned char * b = buf; -my_memcpy( &(R_params.isis), b, 20); -my_memcpy( &(R_params.dplat), b, 10); -my_memcpy( &(R_params.obstype), b, 10); +my_memcpy( &(R_params.isis), b, rad_isis_len); +my_memcpy( &(R_params.dplat), b, rad_dplat_len); +my_memcpy( &(R_params.obstype), b, rad_obstype_len); -my_memcpy( &(R_params.jiter), b, 4); -my_memcpy( &(R_params.nchanl), b, 4); -my_memcpy( &(R_params.npred), b, 4); -my_memcpy( &(R_params.idate), b, 4); -my_memcpy( &(R_params.ireal), b, 4); -my_memcpy( &(R_params.ipchan), b, 4); -my_memcpy( &(R_params.iextra), b, 4); -my_memcpy( &(R_params.jextra), b, 4); +my_memcpy( &(R_params.jiter), b, rad_int_len); +my_memcpy( &(R_params.nchanl), b, rad_int_len); +my_memcpy( &(R_params.npred), b, rad_int_len); +my_memcpy( &(R_params.idate), b, rad_int_len); +my_memcpy( &(R_params.ireal), b, rad_int_len); +my_memcpy( &(R_params.ipchan), b, rad_int_len); +my_memcpy( &(R_params.iextra), b, rad_int_len); +my_memcpy( &(R_params.jextra), b, rad_int_len); -my_memcpy( &(R_params.idiag), b, 4); -my_memcpy( &(R_params.angord), b, 4); -my_memcpy( &(R_params.iversion), b, 4); -my_memcpy( &(R_params.inewpc), b, 4); +my_memcpy( &(R_params.idiag), b, rad_int_len); +my_memcpy( &(R_params.angord), b, rad_int_len); +my_memcpy( &(R_params.iversion), b, rad_int_len); +my_memcpy( &(R_params.inewpc), b, rad_int_len); if ( SwapEndian ) { @@ -483,14 +483,14 @@ if ( n_read != 32 ) { unsigned char * b = buf; -my_memcpy( &(cp.freq), b, 4); -my_memcpy( &(cp.plo), b, 4); -my_memcpy( &(cp.wave), b, 4); -my_memcpy( &(cp.varch), b, 4); -my_memcpy( &(cp.tlap), b, 4); -my_memcpy( &(cp.iuse), b, 4); -my_memcpy( &(cp.nuchan), b, 4); -my_memcpy( &(cp.ich), b, 4); +my_memcpy( &(cp.freq), b, rad_int_len); +my_memcpy( &(cp.plo), b, rad_int_len); +my_memcpy( &(cp.wave), b, rad_int_len); +my_memcpy( &(cp.varch), b, rad_int_len); +my_memcpy( &(cp.tlap), b, rad_int_len); +my_memcpy( &(cp.iuse), b, rad_int_len); +my_memcpy( &(cp.nuchan), b, rad_int_len); +my_memcpy( &(cp.ich), b, rad_int_len); if ( SwapEndian ) { diff --git a/met/src/tools/other/gsi_tools/rad_record.h b/met/src/tools/other/gsi_tools/rad_record.h index c26d768514..f13ac496c8 100644 --- a/met/src/tools/other/gsi_tools/rad_record.h +++ b/met/src/tools/other/gsi_tools/rad_record.h @@ -22,14 +22,21 @@ #include "vx_cal.h" +//////////////////////////////////////////////////////////////////////// + +static const int rad_obstype_len = 10; +static const int rad_dplat_len = 10; +static const int rad_isis_len = 20; +static const int rad_int_len = 4; + //////////////////////////////////////////////////////////////////////// struct RadParams { - char obstype [11]; - char dplat [11]; - char isis [21]; + char obstype [rad_obstype_len+1]; + char dplat [rad_dplat_len + 1]; + char isis [rad_isis_len + 1]; int jiter; int nchanl; From 6260335c389edf937894d1a3d2100d1785596261 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 9 May 2022 22:06:25 -0600 Subject: [PATCH 05/10] #14 Added tmp_buf_size --- met/src/basic/vx_util/util_constants.h | 1 + 1 file changed, 1 insertion(+) diff --git a/met/src/basic/vx_util/util_constants.h b/met/src/basic/vx_util/util_constants.h index 4a40628fa3..b59e7dc46a 100644 --- a/met/src/basic/vx_util/util_constants.h +++ b/met/src/basic/vx_util/util_constants.h @@ -93,6 +93,7 @@ static const int boot_perc_flag = 1; //////////////////////////////////////////////////////////////////////// static const int max_line_len = 2048; +static const int tmp_buf_size = 512; static const double grib_earth_radius_km = 6371.20; static const int default_nc_compression = 0; static const int default_precision = 5; From ffa56ba689bd46051a04a64fabf5265f0f6b0876 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 9 May 2022 22:08:02 -0600 Subject: [PATCH 06/10] #14 Define n_kw_infos first --- met/src/libcode/vx_afm/afm_keywords.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/met/src/libcode/vx_afm/afm_keywords.h b/met/src/libcode/vx_afm/afm_keywords.h index 026f0ca0c5..70c141bdef 100644 --- a/met/src/libcode/vx_afm/afm_keywords.h +++ b/met/src/libcode/vx_afm/afm_keywords.h @@ -83,7 +83,13 @@ struct AfmKeywordInfo { //////////////////////////////////////////////////////////////////////// -static const AfmKeywordInfo kw_info[42] = { +static const int n_kw_infos = 42; + + +//////////////////////////////////////////////////////////////////////// + + +static const AfmKeywordInfo kw_info[n_kw_infos] = { { "StartFontMetrics", afm_keyword_StartFontMetrics }, { "EndFontMetrics", afm_keyword_EndFontMetrics }, @@ -134,12 +140,6 @@ static const AfmKeywordInfo kw_info[42] = { //////////////////////////////////////////////////////////////////////// -static const int n_kw_infos = 42; - - -//////////////////////////////////////////////////////////////////////// - - extern int is_afm_keyword (ConcatString text, AfmKeyword &); From 6d448daaceafa0a3f7fcdbb575f9463e97d8e6c4 Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 9 May 2022 22:08:52 -0600 Subject: [PATCH 07/10] #14 Using literal instead of numbers --- met/src/basic/vx_util/observation.h | 5 +++-- met/src/libcode/vx_color/color.h | 3 ++- met/src/libcode/vx_color/color_table.cc | 6 +++--- met/src/libcode/vx_pxm/ppm.cc | 4 ++-- met/src/libcode/vx_render/ps_filter.cc | 4 +++- met/src/libcode/vx_render/ps_filter.h | 3 ++- met/src/libcode/vx_summary/summary_obs.h | 7 ++++--- met/src/libcode/vx_time_series/compute_swinging_door.h | 2 +- 8 files changed, 20 insertions(+), 14 deletions(-) diff --git a/met/src/basic/vx_util/observation.h b/met/src/basic/vx_util/observation.h index f9230e2dfe..58d9e29285 100644 --- a/met/src/basic/vx_util/observation.h +++ b/met/src/basic/vx_util/observation.h @@ -22,6 +22,7 @@ #include #include "config.h" +#include "util_constants.h" #ifdef ENABLE_PYTHON #include "vx_python3_utils.h" @@ -223,7 +224,7 @@ class Observation { struct tm *time_struct = gmtime(&unix_time); - char time_string[80]; + char time_string[tmp_buf_size]; snprintf(time_string, sizeof(time_string), "%04d%02d%02d_%02d%02d%02d", @@ -240,7 +241,7 @@ class Observation string start_time_string = _getTimeString(start_time); string end_time_string = _getTimeString(end_time); - char time_string[80]; + char time_string[tmp_buf_size]; snprintf(time_string, sizeof(time_string), "%s-%s", diff --git a/met/src/libcode/vx_color/color.h b/met/src/libcode/vx_color/color.h index 746b3610bb..986287aa6c 100644 --- a/met/src/libcode/vx_color/color.h +++ b/met/src/libcode/vx_color/color.h @@ -200,6 +200,7 @@ extern ostream & operator<<(ostream &, const CtableEntry &); static const int ctable_alloc_inc = 30; +static const int fudge_size = 256; //////////////////////////////////////////////////////////////////////// @@ -226,7 +227,7 @@ class ColorTable { double Gamma; - unsigned char fudge[256]; + unsigned char fudge[fudge_size]; public: diff --git a/met/src/libcode/vx_color/color_table.cc b/met/src/libcode/vx_color/color_table.cc index ed6f66843b..46157de13f 100644 --- a/met/src/libcode/vx_color/color_table.cc +++ b/met/src/libcode/vx_color/color_table.cc @@ -332,7 +332,7 @@ Nalloc = 0; int j; -for (j=0; j<256; ++j) { +for (j=0; j max_decimal_places) ) { DecimalPlaces = k; -snprintf(double_format, sizeof(double_format), "%%.%df", DecimalPlaces); // example: "%.5f" +int buf_size = sizeof(double_format); +if (buf_size > filter_buf_size) buf_size = filter_buf_size; +snprintf(double_format, buf_size, "%%.%df", DecimalPlaces); // example: "%.5f" return; diff --git a/met/src/libcode/vx_render/ps_filter.h b/met/src/libcode/vx_render/ps_filter.h index 23b3100de4..00bacc0e25 100644 --- a/met/src/libcode/vx_render/ps_filter.h +++ b/met/src/libcode/vx_render/ps_filter.h @@ -44,6 +44,7 @@ static const int FlateEncode = 4; static const int max_filters = 10; +static const int filter_buf_size = 32; //////////////////////////////////////////////////////////////////////// @@ -60,7 +61,7 @@ class PSFilter { int DecimalPlaces; - char double_format[32]; + char double_format[filter_buf_size]; virtual void eat(unsigned char); diff --git a/met/src/libcode/vx_summary/summary_obs.h b/met/src/libcode/vx_summary/summary_obs.h index 67b2aa353f..d83ef68018 100644 --- a/met/src/libcode/vx_summary/summary_obs.h +++ b/met/src/libcode/vx_summary/summary_obs.h @@ -19,6 +19,7 @@ #include "observation.h" #include "summary_calc.h" #include "time_summary_interval.h" +#include "util_constants.h" //////////////////////////////////////////////////////////////////////// @@ -135,7 +136,7 @@ class SummaryObs // Create the string - char string_buffer[20]; + char string_buffer[tmp_buf_size]; snprintf(string_buffer, sizeof(string_buffer), "%02d%02d%02d", hour, minute, second); @@ -222,7 +223,7 @@ class SummaryObs { struct tm *time_struct = gmtime(&unix_time); - char time_string[80]; + char time_string[tmp_buf_size]; snprintf(time_string, sizeof(time_string), "%04d%02d%02d_%02d%02d%02d", @@ -239,7 +240,7 @@ class SummaryObs memset(&time_struct, 0, sizeof(time_struct)); time_struct.tm_year = atoi(time_string.substr(0, 4).c_str()) - 1900; - time_struct.tm_mon = atoi(time_string.substr(4, 2).c_str()) - 1; + time_struct.tm_mon = atoi(time_string.substr(4, 2).c_str()) - 1; time_struct.tm_mday = atoi(time_string.substr(6, 2).c_str()); time_struct.tm_hour = atoi(time_string.substr(9, 2).c_str()); time_struct.tm_min = atoi(time_string.substr(11, 2).c_str()); diff --git a/met/src/libcode/vx_time_series/compute_swinging_door.h b/met/src/libcode/vx_time_series/compute_swinging_door.h index 9c990fe231..052ce0a789 100644 --- a/met/src/libcode/vx_time_series/compute_swinging_door.h +++ b/met/src/libcode/vx_time_series/compute_swinging_door.h @@ -118,7 +118,7 @@ class SDObservation { struct tm *time_struct = gmtime(&unix_time); - char time_string[80]; + char time_string[tmp_buf_size]; snprintf(time_string, sizeof(time_string), "%04d%02d%02d_%02d%02d%02d", From cc7b0a84b838c1be239f6adca2b32379933f2d1d Mon Sep 17 00:00:00 2001 From: johnhg Date: Tue, 10 May 2022 09:49:24 -0600 Subject: [PATCH 08/10] Bugfix #2148 main_v10.1 misses (#2149) Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> --- met/src/libcode/vx_tc_util/pair_data_genesis.cc | 8 +++++++- met/src/libcode/vx_tc_util/pair_data_genesis.h | 3 ++- met/src/tools/tc_utils/tc_gen/tc_gen.cc | 4 +++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/met/src/libcode/vx_tc_util/pair_data_genesis.cc b/met/src/libcode/vx_tc_util/pair_data_genesis.cc index 2b3e5f07b7..b07185e4c1 100644 --- a/met/src/libcode/vx_tc_util/pair_data_genesis.cc +++ b/met/src/libcode/vx_tc_util/pair_data_genesis.cc @@ -260,7 +260,8 @@ void PairDataGenesis::add_fcst_gen(const GenesisInfo *fgi) { void PairDataGenesis::add_best_gen(const GenesisInfo *bgi, const int fcst_beg, const int fcst_end, const int init_add, const unixtime init_beg, const unixtime init_end, - const TimeArray &init_inc, const TimeArray &init_exc) { + const TimeArray &init_inc, const TimeArray &init_exc, + const NumArray &init_hour, const NumArray &lead) { if(!bgi) return; @@ -282,6 +283,11 @@ void PairDataGenesis::add_best_gen(const GenesisInfo *bgi, (init_exc.n() > 0 && init_exc.has(init_ut))) continue; + // Check if this initialization hour and lead time should be used + if((init_hour.n() > 0 && !init_hour.has(unix_to_sec_of_day(init_ut))) || + (lead.n() > 0 && !lead.has(nint(bgi->genesis_time() - init_ut)))) + continue; + // Check if this case already exists if(!has_case(bgi->storm_id(), init_ut, i_case)) { diff --git a/met/src/libcode/vx_tc_util/pair_data_genesis.h b/met/src/libcode/vx_tc_util/pair_data_genesis.h index 12949ec790..1725273751 100644 --- a/met/src/libcode/vx_tc_util/pair_data_genesis.h +++ b/met/src/libcode/vx_tc_util/pair_data_genesis.h @@ -129,7 +129,8 @@ class PairDataGenesis { void add_best_gen(const GenesisInfo *, const int, const int, const int, const unixtime, const unixtime, - const TimeArray &, const TimeArray &); + const TimeArray &, const TimeArray &, + const NumArray &, const NumArray &); void add_gen_pair(const GenesisInfo *, const GenesisInfo *); void set_gen_diff(int, const GenesisPairDiff &); diff --git a/met/src/tools/tc_utils/tc_gen/tc_gen.cc b/met/src/tools/tc_utils/tc_gen/tc_gen.cc index c6f136d5d5..1e339dbec5 100644 --- a/met/src/tools/tc_utils/tc_gen/tc_gen.cc +++ b/met/src/tools/tc_utils/tc_gen/tc_gen.cc @@ -22,6 +22,7 @@ // 005 04/02/21 Halley Gotway MET #1714 Refinem matching logic // 006 11/04/21 Halley Gotway MET #1809 Add -edeck option // 007 11/22/21 Halley Gotway MET #1810 Add -shape option +// 008 05/02/22 Halley Gotway MET #2148 Fix init_hour and lead misses // //////////////////////////////////////////////////////////////////////// @@ -605,7 +606,8 @@ void get_genesis_pairs(const TCGenVxOpt &vx_opt, conf_info.FcstSecBeg, conf_info.FcstSecEnd, conf_info.InitFreqHr*sec_per_hour, vx_opt.InitBeg, vx_opt.InitEnd, - vx_opt.InitInc, vx_opt.InitExc); + vx_opt.InitInc, vx_opt.InitExc, + vx_opt.InitHour, vx_opt.Lead); } // end for i bga // Loop over the model genesis events looking for pairs. From 44cd88d80675c33e0e4ce52709e91faf164f0d1f Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Tue, 10 May 2022 14:38:42 -0600 Subject: [PATCH 09/10] dtcenter/METplus-Internal#14 formatting code --- met/src/libcode/vx_summary/summary_obs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/met/src/libcode/vx_summary/summary_obs.h b/met/src/libcode/vx_summary/summary_obs.h index d83ef68018..284b67b388 100644 --- a/met/src/libcode/vx_summary/summary_obs.h +++ b/met/src/libcode/vx_summary/summary_obs.h @@ -243,8 +243,8 @@ class SummaryObs time_struct.tm_mon = atoi(time_string.substr(4, 2).c_str()) - 1; time_struct.tm_mday = atoi(time_string.substr(6, 2).c_str()); time_struct.tm_hour = atoi(time_string.substr(9, 2).c_str()); - time_struct.tm_min = atoi(time_string.substr(11, 2).c_str()); - time_struct.tm_sec = atoi(time_string.substr(13, 2).c_str()); + time_struct.tm_min = atoi(time_string.substr(11, 2).c_str()); + time_struct.tm_sec = atoi(time_string.substr(13, 2).c_str()); return timegm(&time_struct); } From 155822206afdb8d5cf779ba402af42c39ce0e392 Mon Sep 17 00:00:00 2001 From: jprestop Date: Mon, 16 May 2022 14:27:39 -0600 Subject: [PATCH 10/10] Feature 2162 v10.1.2 (#2163) * Per #2162, update versions and notes for the 10.1.2 bugfix release * Per #2162, update versions and notes for the 10.1.2 bugfix release * Per #2162, fixing syntax error * Add "METplus-Internal" before the issue number for clarity Co-authored-by: johnhg Co-authored-by: Julie Prestopnik Co-authored-by: johnhg --- met/docs/Users_Guide/release-notes.rst | 9 +++++++++ met/docs/conf.py | 4 ++-- met/src/basic/vx_util/util_constants.h | 3 ++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/met/docs/Users_Guide/release-notes.rst b/met/docs/Users_Guide/release-notes.rst index 9252713b88..5d3385f0cd 100644 --- a/met/docs/Users_Guide/release-notes.rst +++ b/met/docs/Users_Guide/release-notes.rst @@ -5,6 +5,15 @@ When applicable, release notes are followed by the GitHub issue number which des enhancement, or new feature (`MET GitHub issues `_). Important issues are listed **in bold** for emphasis. +MET Version 10.1.2 release notes (20220516) +------------------------------------------- + +* Bugfixes: + + * Update static arrays with hard-coded size (`METplus-Internal #14 `_). + * Fix TC-Gen to only count misses from requested initialization hours and lead times (`#2148 `_). + + MET Version 10.1.1 release notes (20220419) ------------------------------------------- diff --git a/met/docs/conf.py b/met/docs/conf.py index ae0069a127..1ad9bb3d32 100644 --- a/met/docs/conf.py +++ b/met/docs/conf.py @@ -20,11 +20,11 @@ project = 'MET' author = 'UCAR/NCAR, NOAA, CSU/CIRA, and CU/CIRES' author_list = 'Newman, K., J. Opatz, T. Jensen, J. Prestopnik, H. Soh, L. Goodrich, B. Brown, R. Bullock, J. Halley Gotway' -version = '10.1.1' +version = '10.1.2' verinfo = version release = f'{version}' release_year = '2022' -release_date = f'{release_year}-04-19' +release_date = f'{release_year}-05-16' copyright = f'{release_year}, {author}' # -- General configuration --------------------------------------------------- diff --git a/met/src/basic/vx_util/util_constants.h b/met/src/basic/vx_util/util_constants.h index b59e7dc46a..2d334de261 100644 --- a/met/src/basic/vx_util/util_constants.h +++ b/met/src/basic/vx_util/util_constants.h @@ -18,6 +18,7 @@ //////////////////////////////////////////////////////////////////////// // Released versions of MET +static const char met_version_10_1_2[] = "V10.1.2"; static const char met_version_10_1_1[] = "V10.1.1"; static const char met_version_10_1_0[] = "V10.1.0"; static const char met_version_10_0_0[] = "V10.0.0"; @@ -41,7 +42,7 @@ static const char met_version_1_1[] = "V1.1"; //////////////////////////////////////////////////////////////////////// -static const char * const met_version = met_version_10_1_1; +static const char * const met_version = met_version_10_1_2; static const char default_met_data_dir[] = "MET_BASE"; static const char txt_file_ext[] = ".txt"; static const char stat_file_ext[] = ".stat";