diff --git a/docs/Users_Guide/mode.rst b/docs/Users_Guide/mode.rst index 5a6503f0f5..491b452002 100644 --- a/docs/Users_Guide/mode.rst +++ b/docs/Users_Guide/mode.rst @@ -118,6 +118,23 @@ The **multivar_intensity_compare_fcst** and **multivar_intensity_compare_obs** c When regridding to the FCST or OBS field (e.g. to_grid = FCST), the first field of the field array is used from the forecast and observation field dictionaries, respectively. All regridding is then done to that grid. Other regrid options described in :ref:`regrid` can also be used as normal. +"file_type" can be set independently for each input in multivariate mode. If not set for an input, MET uses file names and file content to determine the type. + +When setting a threshold to a percentile, some choices require both an observation input and a forecast input. When this is the case, it's assumed the indices match, so for example if forecast input 1 has such a percentile setting, then observation input 1 will be used to compute the percentile. Percentiles in which this will happen are: + +* SFP in an observation input. + * The matching forecast input will be used to determine the threshold. e.g. ">SFP33.3" in the 2nd observation input means greater than 33.3-rd percentile of the 2nd forecast input will be used as the threshold for that observation input. + +* SOP in a forecast input. + * The matching observation input will be used to determine the threshold. e.g. ">SOP33.3" in the 2nd forecast input means greater than 33.3-rd percentile of the 2nd observation input will be used as the threshold for that forecast input. + +* "==FBIAS" in an observation input. + * e.g. "==FBIAS1" in an observation input to automatically de-bias the data, using a simple threshold in the matching forecast input. For example, when observation input 3 has "==FBIAS1", and forecast input 3 has ">5.0", MET applies the >5.0 threshold to the forecast and then chooses an observation threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0. + +* "==FBIAS" in a forecast input. + * e.g. "==FBIAS1" in a forecast input to automatically de-bias the data, using a simple threshold in the matching observation input. For example, when forecast input 2 has "==FBIAS1", and observation input 2 has ">5.0", MET applies the >5.0 threshold to the observation and then chooses a forecast threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0. + + Practical Information ===================== @@ -260,7 +277,7 @@ _____________________ multivar_name = "Super"; -The **multivar_name** entry appears only in the **MODEMultivarConfig_default** file. This option is used only when the multivar option is enabled, and only when all **multivar_intensity_flag** values are FALSE. It can be thought of as an identifier for the multivariate super object. It shows up in output files names and content. It can be set separately for forecasts and observations or as a common value for both. +The **multivar_name** entry appears only in the **MODEMultivarConfig_default** file. This option is used only when the multivar option is enabled, and only when **multivar_intensity_compare_fcst** and **multivar_intensity_compare_obs** are empty. It can be thought of as an identifier for the multivariate super object. It shows up in output files names and content. It can be set separately for forecasts and observations or as a common value for both. _____________________ @@ -268,7 +285,7 @@ _____________________ multivar_level = "LO"; -The **multivar_level** entry appears only in the **MODEMultivarConfig_default** file. This option is used only when the multivar option is enabled, and only when all **multivar_intensity_flag** values are FALSE. It is the identifier for the multivariate super object as regards level. It shows up in output files names and content. If not set the default value is "NA". It can be set separately for forecasts and observations, or as a common value for both. +The **multivar_level** entry appears only in the **MODEMultivarConfig_default** file. This option is used only when the multivar option is enabled, and only when **multivar_intensity_compare_fcst** and **multivar_intensity_compare_obs** are empty. It is the identifier for the multivariate super object as regards level. It shows up in output files names and content. If not set the default value is "NA". It can be set separately for forecasts and observations, or as a common value for both. _____________________ diff --git a/internal/test_unit/config/MODEConfig_multivar b/internal/test_unit/config/MODEConfig_multivar new file mode 100644 index 0000000000..d3fba7556d --- /dev/null +++ b/internal/test_unit/config/MODEConfig_multivar @@ -0,0 +1,274 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// MODE configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +// model = +model = "HRRR"; + +// +// Output description to be written +// +// desc = + +// +// Output observation type to be written +// +// obtype = +obtype = "ANALYSIS"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = {to_grid = FCST;method = NEAREST;width = 1;vld_thresh = 0.5;} + +//////////////////////////////////////////////////////////////////////////////// + +// +`// Approximate grid resolution (km) +// +// grid_res = +// + +//////////////////////////////////////////////////////////////////////////////// + +// +// Run all permutations of radius and threshold +// +// quilt = +// ${METPLUS_QUILT} + +// +// MODE Multivar boolean combination logic +// +//multivar_logic = +//${METPLUS_MULTIVAR_LOGIC} +multivar_logic = "#1 && #2 && #3"; + +// keep this around to compare to older versions, the new version doesn't read this +multivar_intensity_flag = [FALSE, TRUE, TRUE]; + +multivar_intensity_compare_fcst = [ 2, 3 ]; +multivar_intensity_compare_obs = [ 2, 3 ]; + +// +// Forecast and observation fields to be verified +// +fcst = { + //${METPLUS_FCST_FIELD} + field = [{ name="CSNOW"; level="L0"; conv_radius = 0; conv_thresh = ==1; file_type = GRIB2; merge_flag = NONE; }, + { name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; }, + { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; }]; + + //${METPLUS_FCST_CENSOR_THRESH} + //${METPLUS_FCST_CENSOR_VAL} + //${METPLUS_FCST_CONV_RADIUS} + //${METPLUS_FCST_CONV_THRESH} + //${METPLUS_FCST_VLD_THRESH} + //${METPLUS_FCST_FILTER_ATTR_NAME} + filter_attr_name = ["AREA"]; + + //${METPLUS_FCST_FILTER_ATTR_THRESH} + filter_attr_thresh = [>=25]; + + //${METPLUS_FCST_MERGE_THRESH} + //${METPLUS_FCST_MERGE_FLAG} + //${METPLUS_FCST_FILE_TYPE} + //file_type = GRIB2; + multivar_name = "Snow"; + multivar_level = "LO"; +} + +obs = { + //${METPLUS_OBS_FIELD} + field = [{ name="PrecipFlag"; level="L0"; conv_radius = 0; conv_thresh = ==3; file_type = GRIB2; merge_flag = NONE; }, + { name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; }, + { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; } ]; + + //${METPLUS_OBS_CENSOR_THRESH} + //${METPLUS_OBS_CENSOR_VAL} + //${METPLUS_OBS_CONV_RADIUS} + //${METPLUS_OBS_CONV_THRESH} + //${METPLUS_OBS_VLD_THRESH} + + //${METPLUS_OBS_FILTER_ATTR_NAME} + filter_attr_name = ["AREA"]; + + //${METPLUS_OBS_FILTER_ATTR_THRESH} + filter_attr_thresh = [>=25]; + + // ${METPLUS_OBS_MERGE_THRESH} + // ${METPLUS_OBS_MERGE_FLAG} + // ${METPLUS_OBS_FILE_TYPE} + multivar_name = "Precip"; + multivar_level = "LO"; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Handle missing data +// +// mask_missing_flag = +// ${METPLUS_MASK_MISSING_FLAG} + +// +// Match objects between the forecast and observation fields +// +//match_flag = +//${METPLUS_MATCH_FLAG} +match_flag = MERGE_BOTH; +// +// Maximum centroid distance for objects to be compared +// +//max_centroid_dist = +//${METPLUS_MAX_CENTROID_DIST} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +//mask = { +//${METPLUS_MASK_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fuzzy engine weights +// +//weight = { +//${METPLUS_WEIGHT_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fuzzy engine interest functions +// +interest_function = { + + //${METPLUS_INTEREST_FUNCTION_CENTROID_DIST} + + //${METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST} + + //${METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST} + + angle_diff = ( + ( 0.0, 1.0 ) + ( 30.0, 1.0 ) + ( 90.0, 0.0 ) + ); + + aspect_diff = ( + ( 0.00, 1.0 ) + ( 0.10, 1.0 ) + ( 0.75, 0.0 ) + ); + + corner = 0.8; + ratio_if = ( + ( 0.0, 0.0 ) + ( corner, 1.0 ) + ( 1.0, 1.0 ) + ); + + area_ratio = ratio_if; + + int_area_ratio = ( + ( 0.00, 0.00 ) + ( 0.10, 0.50 ) + ( 0.25, 1.00 ) + ( 1.00, 1.00 ) + ); + + curvature_ratio = ratio_if; + + complexity_ratio = ratio_if; + + inten_perc_ratio = ratio_if; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Total interest threshold for determining matches +// +//total_interest_thresh = +//${METPLUS_TOTAL_INTEREST_THRESH} + +// +// Interest threshold for printing output pair information +// +print_interest_thresh = 0.0; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Plotting information +// +met_data_dir = "MET_BASE"; + +fcst_raw_plot = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; +} + +obs_raw_plot = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; +} + +object_plot = { + color_table = "MET_BASE/colortables/mode_obj.ctable"; +} + +// +// Boolean for plotting on the region of valid data within the domain +// +plot_valid_flag = FALSE; + +// +// Plot polyline edges using great circle arcs instead of straight lines +// +plot_gcarc_flag = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// NetCDF matched pairs, PostScript, and contingency table output files +// +//ps_plot_flag = +//${METPLUS_PS_PLOT_FLAG} + +//nc_pairs_flag = { +//${METPLUS_NC_PAIRS_FLAG_DICT} + +//ct_stats_flag = +//${METPLUS_CT_STATS_FLAG} + + +//////////////////////////////////////////////////////////////////////////////// + +shift_right = 0; // grid squares + +//////////////////////////////////////////////////////////////////////////////// + +//${METPLUS_OUTPUT_PREFIX} +//version = "V10.0"; + +//tmp_dir = "${MET_TMP_DIR}"; +tmp_dir = "/tmp"; + +//////////////////////////////////////////////////////////////////////////////// + +//${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/internal/test_unit/config/MODEConfig_multivar_3_2 b/internal/test_unit/config/MODEConfig_multivar_3_2 new file mode 100644 index 0000000000..716b93c9be --- /dev/null +++ b/internal/test_unit/config/MODEConfig_multivar_3_2 @@ -0,0 +1,275 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Mode configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +// model = +model = "HRRR"; + +// +// Output description to be written +// +// desc = + +// +// Output observation type to be written +// +// obtype = +obtype = "ANALYSIS"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +// regrid = { +regrid = {to_grid = FCST;method = NEAREST;width = 1;vld_thresh = 0.5;} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Approximate grid resolution (km) +// +// grid_res = +// + +//////////////////////////////////////////////////////////////////////////////// + +// +// Run all permutations of radius and threshold +// +// quilt = +// ${METPLUS_QUILT} + +// +// MODE Multivar boolean combination logic +// +//multivar_logic = +//${METPLUS_MULTIVAR_LOGIC} +multivar_logic = "#1 && #2 && #3"; + +multivar_intensity_compare_fcst = [ 2, 2 ]; +multivar_intensity_compare_obs = [ 1, 2 ]; + +// +// Forecast and observation fields to be verified +// +fcst = { + //${METPLUS_FCST_FIELD} + field = [{ name="CSNOW"; level="L0"; conv_radius = 0; conv_thresh = ==1; file_type = GRIB2; merge_flag = NONE; }, + { name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; }, + { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; } ]; + + multivar_logic = "#1 && #2 && #3"; + + //${METPLUS_FCST_CENSOR_THRESH} + //${METPLUS_FCST_CENSOR_VAL} + //${METPLUS_FCST_CONV_RADIUS} + //${METPLUS_FCST_CONV_THRESH} + //${METPLUS_FCST_VLD_THRESH} + //${METPLUS_FCST_FILTER_ATTR_NAME} + filter_attr_name = ["AREA"]; + + //${METPLUS_FCST_FILTER_ATTR_THRESH} + filter_attr_thresh = [>=25]; + + //${METPLUS_FCST_MERGE_THRESH} + //${METPLUS_FCST_MERGE_FLAG} + //${METPLUS_FCST_FILE_TYPE} + //file_type = GRIB2; + multivar_name = "Snow"; + multivar_level = "LO"; +} + +obs = { + //${METPLUS_OBS_FIELD} + field = [{ name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; }, + { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; } ]; + + multivar_logic = "#1 && #2"; + + //${METPLUS_OBS_CENSOR_THRESH} + //${METPLUS_OBS_CENSOR_VAL} + //${METPLUS_OBS_CONV_RADIUS} + //${METPLUS_OBS_CONV_THRESH} + //${METPLUS_OBS_VLD_THRESH} + + //${METPLUS_OBS_FILTER_ATTR_NAME} + filter_attr_name = ["AREA"]; + + //${METPLUS_OBS_FILTER_ATTR_THRESH} + filter_attr_thresh = [>=25]; + + // ${METPLUS_OBS_MERGE_THRESH} + // ${METPLUS_OBS_MERGE_FLAG} + // ${METPLUS_OBS_FILE_TYPE} + multivar_name = "Precip"; + multivar_level = "LO"; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Handle missing data +// +// mask_missing_flag = +// ${METPLUS_MASK_MISSING_FLAG} + +// +// Match objects between the forecast and observation fields +// +//match_flag = +//${METPLUS_MATCH_FLAG} +match_flag = MERGE_BOTH; +// +// Maximum centroid distance for objects to be compared +// +//max_centroid_dist = +//${METPLUS_MAX_CENTROID_DIST} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +//mask = { +//${METPLUS_MASK_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fuzzy engine weights +// +//weight = { +//${METPLUS_WEIGHT_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fuzzy engine interest functions +// +interest_function = { + + //${METPLUS_INTEREST_FUNCTION_CENTROID_DIST} + + //${METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST} + + //${METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST} + + angle_diff = ( + ( 0.0, 1.0 ) + ( 30.0, 1.0 ) + ( 90.0, 0.0 ) + ); + + aspect_diff = ( + ( 0.00, 1.0 ) + ( 0.10, 1.0 ) + ( 0.75, 0.0 ) + ); + + corner = 0.8; + ratio_if = ( + ( 0.0, 0.0 ) + ( corner, 1.0 ) + ( 1.0, 1.0 ) + ); + + area_ratio = ratio_if; + + int_area_ratio = ( + ( 0.00, 0.00 ) + ( 0.10, 0.50 ) + ( 0.25, 1.00 ) + ( 1.00, 1.00 ) + ); + + curvature_ratio = ratio_if; + + complexity_ratio = ratio_if; + + inten_perc_ratio = ratio_if; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Total interest threshold for determining matches +// +//total_interest_thresh = +//${METPLUS_TOTAL_INTEREST_THRESH} + +// +// Interest threshold for printing output pair information +// +print_interest_thresh = 0.0; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Plotting information +// +met_data_dir = "MET_BASE"; + +fcst_raw_plot = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; +} + +obs_raw_plot = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; +} + +object_plot = { + color_table = "MET_BASE/colortables/mode_obj.ctable"; +} + +// +// Boolean for plotting on the region of valid data within the domain +// +plot_valid_flag = FALSE; + +// +// Plot polyline edges using great circle arcs instead of straight lines +// +plot_gcarc_flag = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// NetCDF matched pairs, PostScript, and contingency table output files +// +//ps_plot_flag = +//${METPLUS_PS_PLOT_FLAG} + +//nc_pairs_flag = { +//${METPLUS_NC_PAIRS_FLAG_DICT} + +//ct_stats_flag = +//${METPLUS_CT_STATS_FLAG} + + +//////////////////////////////////////////////////////////////////////////////// + +shift_right = 0; // grid squares + +//////////////////////////////////////////////////////////////////////////////// + +//${METPLUS_OUTPUT_PREFIX} +//version = "V10.0"; + +//tmp_dir = "${MET_TMP_DIR}"; +tmp_dir = "/tmp"; + +//////////////////////////////////////////////////////////////////////////////// + +//${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/internal/test_unit/config/MODEConfig_multivar_super b/internal/test_unit/config/MODEConfig_multivar_super new file mode 100644 index 0000000000..e035f6ba63 --- /dev/null +++ b/internal/test_unit/config/MODEConfig_multivar_super @@ -0,0 +1,271 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// MODE configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +// model = +model = "HRRR"; + +// +// Output description to be written +// +// desc = + +// +// Output observation type to be written +// +// obtype = +obtype = "ANALYSIS"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = {to_grid = FCST;method = NEAREST;width = 1;vld_thresh = 0.5;} + +//////////////////////////////////////////////////////////////////////////////// + +// +`// Approximate grid resolution (km) +// +// grid_res = +// + +//////////////////////////////////////////////////////////////////////////////// + +// +// Run all permutations of radius and threshold +// +// quilt = +// ${METPLUS_QUILT} + +// +// MODE Multivar boolean combination logic +// +//multivar_logic = +//${METPLUS_MULTIVAR_LOGIC} +multivar_logic = "#1 && #2 && #3"; + +multivar_intensity_compare_fcst = [ ]; +multivar_intensity_compare_obs = [ ]; + +// +// Forecast and observation fields to be verified +// +fcst = { + //${METPLUS_FCST_FIELD} + field = [{ name="CSNOW"; level="L0"; conv_radius = 0; conv_thresh = ==1; file_type = GRIB2; merge_flag = NONE; }, + { name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; }, + { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; } ]; + + //${METPLUS_FCST_CENSOR_THRESH} + //${METPLUS_FCST_CENSOR_VAL} + //${METPLUS_FCST_CONV_RADIUS} + //${METPLUS_FCST_CONV_THRESH} + //${METPLUS_FCST_VLD_THRESH} + //${METPLUS_FCST_FILTER_ATTR_NAME} + filter_attr_name = ["AREA"]; + + //${METPLUS_FCST_FILTER_ATTR_THRESH} + filter_attr_thresh = [>=25]; + + //${METPLUS_FCST_MERGE_THRESH} + //${METPLUS_FCST_MERGE_FLAG} + //${METPLUS_FCST_FILE_TYPE} + //file_type = GRIB2; + multivar_name = "Snow"; + multivar_level = "LO"; +} + +obs = { + //${METPLUS_OBS_FIELD} + field = [{ name="PrecipFlag"; level="L0"; conv_radius = 0; conv_thresh = ==3; file_type = GRIB2; merge_flag = NONE; }, + { name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; }, + { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; } ]; + + //${METPLUS_OBS_CENSOR_THRESH} + //${METPLUS_OBS_CENSOR_VAL} + //${METPLUS_OBS_CONV_RADIUS} + //${METPLUS_OBS_CONV_THRESH} + //${METPLUS_OBS_VLD_THRESH} + + //${METPLUS_OBS_FILTER_ATTR_NAME} + filter_attr_name = ["AREA"]; + + //${METPLUS_OBS_FILTER_ATTR_THRESH} + filter_attr_thresh = [>=25]; + + // ${METPLUS_OBS_MERGE_THRESH} + // ${METPLUS_OBS_MERGE_FLAG} + // ${METPLUS_OBS_FILE_TYPE} + multivar_name = "Precip"; + multivar_level = "LO"; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Handle missing data +// +// mask_missing_flag = +// ${METPLUS_MASK_MISSING_FLAG} + +// +// Match objects between the forecast and observation fields +// +//match_flag = +//${METPLUS_MATCH_FLAG} +match_flag = MERGE_BOTH; +// +// Maximum centroid distance for objects to be compared +// +//max_centroid_dist = +//${METPLUS_MAX_CENTROID_DIST} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +//mask = { +//${METPLUS_MASK_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fuzzy engine weights +// +//weight = { +//${METPLUS_WEIGHT_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fuzzy engine interest functions +// +interest_function = { + + //${METPLUS_INTEREST_FUNCTION_CENTROID_DIST} + + //${METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST} + + //${METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST} + + angle_diff = ( + ( 0.0, 1.0 ) + ( 30.0, 1.0 ) + ( 90.0, 0.0 ) + ); + + aspect_diff = ( + ( 0.00, 1.0 ) + ( 0.10, 1.0 ) + ( 0.75, 0.0 ) + ); + + corner = 0.8; + ratio_if = ( + ( 0.0, 0.0 ) + ( corner, 1.0 ) + ( 1.0, 1.0 ) + ); + + area_ratio = ratio_if; + + int_area_ratio = ( + ( 0.00, 0.00 ) + ( 0.10, 0.50 ) + ( 0.25, 1.00 ) + ( 1.00, 1.00 ) + ); + + curvature_ratio = ratio_if; + + complexity_ratio = ratio_if; + + inten_perc_ratio = ratio_if; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Total interest threshold for determining matches +// +//total_interest_thresh = +//${METPLUS_TOTAL_INTEREST_THRESH} + +// +// Interest threshold for printing output pair information +// +print_interest_thresh = 0.0; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Plotting information +// +met_data_dir = "MET_BASE"; + +fcst_raw_plot = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; +} + +obs_raw_plot = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; +} + +object_plot = { + color_table = "MET_BASE/colortables/mode_obj.ctable"; +} + +// +// Boolean for plotting on the region of valid data within the domain +// +plot_valid_flag = FALSE; + +// +// Plot polyline edges using great circle arcs instead of straight lines +// +plot_gcarc_flag = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// NetCDF matched pairs, PostScript, and contingency table output files +// +//ps_plot_flag = +//${METPLUS_PS_PLOT_FLAG} + +//nc_pairs_flag = { +//${METPLUS_NC_PAIRS_FLAG_DICT} + +//ct_stats_flag = +//${METPLUS_CT_STATS_FLAG} + + +//////////////////////////////////////////////////////////////////////////////// + +shift_right = 0; // grid squares + +//////////////////////////////////////////////////////////////////////////////// + +//${METPLUS_OUTPUT_PREFIX} +//version = "V10.0"; + +//tmp_dir = "${MET_TMP_DIR}"; +tmp_dir = "/tmp"; + +//////////////////////////////////////////////////////////////////////////////// + +//${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/internal/test_unit/xml/unit_mode_multivar.xml b/internal/test_unit/xml/unit_mode_multivar.xml index 4aad5dc7d6..aea7f7f2a0 100644 --- a/internal/test_unit/xml/unit_mode_multivar.xml +++ b/internal/test_unit/xml/unit_mode_multivar.xml @@ -16,7 +16,90 @@ &TEST_DIR; true - + + echo "&DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2 \ + &DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2 \ + &DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2" \ + > &OUTPUT_DIR;/mode_multivar/input_fcst_file_list; \ + echo "&DATA_DIR_OBS;/mode_multivar/mrms/20210201/PrecipFlag_00.00_20210201-210000.sub.grib2 \ + &DATA_DIR_OBS;/mode_multivar/hrrr/anl/20210201/hrrr.t21z.wrfprsf00.sub.grib2 \ + &DATA_DIR_OBS;/mode_multivar/hrrr/anl/20210201/hrrr.t21z.wrfprsf00.sub.grib2" \ + > &OUTPUT_DIR;/mode_multivar/input_obs_file_list; \ + &MET_BIN;/mode + \ + &OUTPUT_DIR;/mode_multivar/input_fcst_file_list \ + &OUTPUT_DIR;/mode_multivar/input_obs_file_list \ + &CONFIG_DIR;/MODEConfig_multivar \ + -outdir &OUTPUT_DIR;/mode_multivar \ + -v 2 + + + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A_obj.nc + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A_obj.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A_cts.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A.ps + &OUTPUT_DIR;/mode_multivar/mode_Fcst_WIND_Z10_Obs_WIND_Z10_210000L_20210201_210000V_000000A_obj.nc + &OUTPUT_DIR;/mode_multivar/mode_Fcst_WIND_Z10_Obs_WIND_Z10_210000L_20210201_210000V_000000A_obj.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_WIND_Z10_Obs_WIND_Z10_210000L_20210201_210000V_000000A_cts.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_WIND_Z10_Obs_WIND_Z10_210000L_20210201_210000V_000000A.ps + + + + + + echo "&DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2 \ + &DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2 \ + &DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2" \ + > &OUTPUT_DIR;/mode_multivar/input_fcst_file_list; \ + echo "&DATA_DIR_OBS;/mode_multivar/hrrr/anl/20210201/hrrr.t21z.wrfprsf00.sub.grib2 \ + &DATA_DIR_OBS;/mode_multivar/hrrr/anl/20210201/hrrr.t21z.wrfprsf00.sub.grib2" \ + > &OUTPUT_DIR;/mode_multivar/input_obs_file_list; \ + &MET_BIN;/mode + \ + &OUTPUT_DIR;/mode_multivar/input_fcst_file_list \ + &OUTPUT_DIR;/mode_multivar/input_obs_file_list \ + &CONFIG_DIR;/MODEConfig_multivar_3_2 \ + -outdir &OUTPUT_DIR;/mode_multivar \ + -v 2 + + + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A_obj.nc + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A_obj.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A_cts.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A.ps + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_WIND_Z10_210000L_20210201_210000V_000000A_obj.nc + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_WIND_Z10_210000L_20210201_210000V_000000A_obj.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_WIND_Z10_210000L_20210201_210000V_000000A_cts.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_WIND_Z10_210000L_20210201_210000V_000000A.ps + + + + + echo "&DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2 \ + &DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2 \ + &DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2" \ + > &OUTPUT_DIR;/mode_multivar/input_fcst_file_list; \ + echo "&DATA_DIR_OBS;/mode_multivar/mrms/20210201/PrecipFlag_00.00_20210201-210000.sub.grib2 \ + &DATA_DIR_OBS;/mode_multivar/hrrr/anl/20210201/hrrr.t21z.wrfprsf00.sub.grib2 \ + &DATA_DIR_OBS;/mode_multivar/hrrr/anl/20210201/hrrr.t21z.wrfprsf00.sub.grib2" \ + > &OUTPUT_DIR;/mode_multivar/input_obs_file_list; \ + &MET_BIN;/mode + \ + &OUTPUT_DIR;/mode_multivar/input_fcst_file_list \ + &OUTPUT_DIR;/mode_multivar/input_obs_file_list \ + &CONFIG_DIR;/MODEConfig_multivar_super \ + -outdir &OUTPUT_DIR;/mode_multivar \ + -v 2 + + + &OUTPUT_DIR;/mode_multivar/mode_Fcst_Snow_LO_Obs_Precip_LO_210000L_20210201_210000V_000000A_obj.nc + &OUTPUT_DIR;/mode_multivar/mode_Fcst_Snow_LO_Obs_Precip_LO_210000L_20210201_210000V_000000A_obj.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_Snow_LO_Obs_Precip_LO_210000L_20210201_210000V_000000A_cts.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_Snow_LO_Obs_Precip_LO_210000L_20210201_210000V_000000A.ps + + + + echo "&DATA_DIR_MODEL;/mode_multivar/alpha_fcst.nc \ &DATA_DIR_MODEL;/mode_multivar/beta_fcst.nc \ &DATA_DIR_MODEL;/mode_multivar/gamma_fcst.nc" \ diff --git a/src/basic/vx_config/config_file.cc b/src/basic/vx_config/config_file.cc index d349b2d140..0ca0bf5aa0 100644 --- a/src/basic/vx_config/config_file.cc +++ b/src/basic/vx_config/config_file.cc @@ -115,6 +115,20 @@ assign(c); } +//////////////////////////////////////////////////////////////////////// + +MetConfig & MetConfig::operator=(const MetConfig &s) +{ + if(this == &s) return(*this); + + init_from_scratch(); + + assign(s); + + return(*this); + +} + //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_data2d/var_info.h b/src/libcode/vx_data2d/var_info.h index 7dcf6e386a..15fa2f90b7 100644 --- a/src/libcode/vx_data2d/var_info.h +++ b/src/libcode/vx_data2d/var_info.h @@ -91,11 +91,14 @@ class VarInfo VarInfo(const VarInfo &); VarInfo & operator=(const VarInfo &); + virtual VarInfo *clone() const = 0; + // Conversion function UserFunc_1Arg ConvertFx; void clear(); - + void clone_base() const; + virtual void dump(std::ostream &) const; // diff --git a/src/libcode/vx_data2d_grib/var_info_grib.cc b/src/libcode/vx_data2d_grib/var_info_grib.cc index c414bc94f8..e6bb74198f 100644 --- a/src/libcode/vx_data2d_grib/var_info_grib.cc +++ b/src/libcode/vx_data2d_grib/var_info_grib.cc @@ -75,6 +75,15 @@ VarInfoGrib & VarInfoGrib::operator=(const VarInfoGrib &f) { /////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoGrib::clone() const { + + VarInfoGrib *ret = new VarInfoGrib(*this); + + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoGrib::init_from_scratch() { // First call the parent's initialization diff --git a/src/libcode/vx_data2d_grib/var_info_grib.h b/src/libcode/vx_data2d_grib/var_info_grib.h index cdf0947b90..d695eed007 100644 --- a/src/libcode/vx_data2d_grib/var_info_grib.h +++ b/src/libcode/vx_data2d_grib/var_info_grib.h @@ -67,6 +67,7 @@ class VarInfoGrib : public VarInfo ~VarInfoGrib(); VarInfoGrib(const VarInfoGrib &); VarInfoGrib & operator=(const VarInfoGrib &); + VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_data2d_grib2/var_info_grib2.cc b/src/libcode/vx_data2d_grib2/var_info_grib2.cc index 6ea70eecfa..ef3a3c7db6 100644 --- a/src/libcode/vx_data2d_grib2/var_info_grib2.cc +++ b/src/libcode/vx_data2d_grib2/var_info_grib2.cc @@ -77,6 +77,15 @@ VarInfoGrib2 & VarInfoGrib2::operator=(const VarInfoGrib2 &f) { /////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoGrib2::clone() const { + + VarInfoGrib2 *ret = new VarInfoGrib2(*this); + + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoGrib2::init_from_scratch() { // First call the parent's initialization diff --git a/src/libcode/vx_data2d_grib2/var_info_grib2.h b/src/libcode/vx_data2d_grib2/var_info_grib2.h index 619508358b..5f8e320339 100644 --- a/src/libcode/vx_data2d_grib2/var_info_grib2.h +++ b/src/libcode/vx_data2d_grib2/var_info_grib2.h @@ -72,6 +72,7 @@ class VarInfoGrib2 : public VarInfo ~VarInfoGrib2(); VarInfoGrib2(const VarInfoGrib2 &); VarInfoGrib2 & operator=(const VarInfoGrib2 &); + VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.cc b/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.cc index 318f4524bd..dba8bd4ec5 100644 --- a/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.cc +++ b/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.cc @@ -76,6 +76,15 @@ VarInfoNcCF & VarInfoNcCF::operator=(const VarInfoNcCF &f) { /////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoNcCF::clone() const { + + VarInfoNcCF *ret = new VarInfoNcCF(*this); + + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoNcCF::init_from_scratch() { // First call the parent's initialization diff --git a/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.h b/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.h index 2463bcbde2..ce20289c2c 100644 --- a/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.h +++ b/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.h @@ -45,6 +45,7 @@ class VarInfoNcCF : public VarInfo ~VarInfoNcCF(); VarInfoNcCF(const VarInfoNcCF &); VarInfoNcCF & operator=(const VarInfoNcCF &); + VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_data2d_nc_met/var_info_nc_met.cc b/src/libcode/vx_data2d_nc_met/var_info_nc_met.cc index 651832d69d..c1624d8785 100644 --- a/src/libcode/vx_data2d_nc_met/var_info_nc_met.cc +++ b/src/libcode/vx_data2d_nc_met/var_info_nc_met.cc @@ -76,6 +76,15 @@ VarInfoNcMet & VarInfoNcMet::operator=(const VarInfoNcMet &f) { /////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoNcMet::clone() const { + + VarInfoNcMet *ret = new VarInfoNcMet(*this); + + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoNcMet::init_from_scratch() { // First call the parent's initialization diff --git a/src/libcode/vx_data2d_nc_met/var_info_nc_met.h b/src/libcode/vx_data2d_nc_met/var_info_nc_met.h index a9f0c66f53..f85bdfdb7d 100644 --- a/src/libcode/vx_data2d_nc_met/var_info_nc_met.h +++ b/src/libcode/vx_data2d_nc_met/var_info_nc_met.h @@ -41,6 +41,7 @@ class VarInfoNcMet : public VarInfo ~VarInfoNcMet(); VarInfoNcMet(const VarInfoNcMet &); VarInfoNcMet & operator=(const VarInfoNcMet &); + virtual VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.cc b/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.cc index eecf2270cb..351ddf302a 100644 --- a/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.cc +++ b/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.cc @@ -72,6 +72,15 @@ VarInfoNcWrf & VarInfoNcWrf::operator=(const VarInfoNcWrf &f) { /////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoNcWrf::clone() const { + + VarInfoNcWrf *ret = new VarInfoNcWrf(*this); + + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoNcWrf::init_from_scratch() { // First call the parent's initialization diff --git a/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.h b/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.h index 182ad62d28..24569fe7e4 100644 --- a/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.h +++ b/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.h @@ -204,6 +204,7 @@ class VarInfoNcWrf : public VarInfo ~VarInfoNcWrf(); VarInfoNcWrf(const VarInfoNcWrf &); VarInfoNcWrf & operator=(const VarInfoNcWrf &); + VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_data2d_python/var_info_python.cc b/src/libcode/vx_data2d_python/var_info_python.cc index 5c111ec28b..74d8767816 100644 --- a/src/libcode/vx_data2d_python/var_info_python.cc +++ b/src/libcode/vx_data2d_python/var_info_python.cc @@ -75,6 +75,15 @@ VarInfoPython & VarInfoPython::operator=(const VarInfoPython &f) { /////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoPython::clone() const { + + VarInfoPython *ret = new VarInfoPython(*this); + + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoPython::init_from_scratch() { // First call the parent's initialization diff --git a/src/libcode/vx_data2d_python/var_info_python.h b/src/libcode/vx_data2d_python/var_info_python.h index 68309f8b4a..e38f9edae2 100644 --- a/src/libcode/vx_data2d_python/var_info_python.h +++ b/src/libcode/vx_data2d_python/var_info_python.h @@ -41,6 +41,7 @@ class VarInfoPython : public VarInfo ~VarInfoPython(); VarInfoPython(const VarInfoPython &); VarInfoPython & operator=(const VarInfoPython &); + VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_data2d_ugrid/var_info_ugrid.cc b/src/libcode/vx_data2d_ugrid/var_info_ugrid.cc index e29fa44c95..9e74762a48 100644 --- a/src/libcode/vx_data2d_ugrid/var_info_ugrid.cc +++ b/src/libcode/vx_data2d_ugrid/var_info_ugrid.cc @@ -32,7 +32,6 @@ using namespace std; - /////////////////////////////////////////////////////////////////////////////// static bool is_grib_code_abbr_match(const ConcatString &, int); @@ -77,6 +76,15 @@ VarInfoUGrid & VarInfoUGrid::operator=(const VarInfoUGrid &f) { /////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoUGrid::clone() const { + + VarInfoUGrid *ret = new VarInfoUGrid(*this); + + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoUGrid::init_from_scratch() { // First call the parent's initialization diff --git a/src/libcode/vx_data2d_ugrid/var_info_ugrid.h b/src/libcode/vx_data2d_ugrid/var_info_ugrid.h index b7c64b1014..e3bb5a1bb9 100644 --- a/src/libcode/vx_data2d_ugrid/var_info_ugrid.h +++ b/src/libcode/vx_data2d_ugrid/var_info_ugrid.h @@ -45,6 +45,7 @@ class VarInfoUGrid : public VarInfo ~VarInfoUGrid(); VarInfoUGrid(const VarInfoUGrid &); VarInfoUGrid & operator=(const VarInfoUGrid &); + VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_shapedata/Makefile.am b/src/libcode/vx_shapedata/Makefile.am index f54e937535..ea52307df3 100644 --- a/src/libcode/vx_shapedata/Makefile.am +++ b/src/libcode/vx_shapedata/Makefile.am @@ -21,6 +21,7 @@ libvx_shapedata_a_SOURCES = \ mode_columns.h \ mode_field_info.cc mode_field_info.h \ mode_conf_info.cc mode_conf_info.h \ + mode_input_data.h \ engine.cc engine.h \ ihull.cc ihull.h \ vx_shapedata.h diff --git a/src/libcode/vx_shapedata/Makefile.in b/src/libcode/vx_shapedata/Makefile.in index df218fceda..290ee7582f 100644 --- a/src/libcode/vx_shapedata/Makefile.in +++ b/src/libcode/vx_shapedata/Makefile.in @@ -369,6 +369,7 @@ libvx_shapedata_a_SOURCES = \ mode_columns.h \ mode_field_info.cc mode_field_info.h \ mode_conf_info.cc mode_conf_info.h \ + mode_input_data.h \ engine.cc engine.h \ ihull.cc ihull.h \ vx_shapedata.h diff --git a/src/libcode/vx_shapedata/engine.cc b/src/libcode/vx_shapedata/engine.cc index e81c2d21bc..6087bf62bd 100644 --- a/src/libcode/vx_shapedata/engine.cc +++ b/src/libcode/vx_shapedata/engine.cc @@ -144,6 +144,7 @@ void ModeFuzzyEngine::init_from_scratch() { // // Reset all fcst and obs processing flags to initial state // + need_fcst_conv = true; need_fcst_thresh = true; need_fcst_filter = true; @@ -964,9 +965,7 @@ void ModeFuzzyEngine::do_obs_merging(const char *default_config, /////////////////////////////////////////////////////////////////////// -void ModeFuzzyEngine::do_fcst_merging(const char *default_config, - const char *merge_config, - const ShapeData &merge_data) +void ModeFuzzyEngine::do_fcst_merging(const ShapeData &merge_data) { if(need_fcst_thresh) do_fcst_thresholding(); @@ -985,7 +984,7 @@ void ModeFuzzyEngine::do_fcst_merging(const char *default_config, if(conf_info.Fcst->merge_flag == MergeType_Both || conf_info.Fcst->merge_flag == MergeType_Engine) - do_fcst_merge_engine(default_config, merge_config); + do_fcst_merge_engine("", ""); // // Done @@ -1000,9 +999,7 @@ void ModeFuzzyEngine::do_fcst_merging(const char *default_config, /////////////////////////////////////////////////////////////////////// -void ModeFuzzyEngine::do_obs_merging(const char *default_config, - const char *merge_config, - const ShapeData &merge_data) +void ModeFuzzyEngine::do_obs_merging(const ShapeData &merge_data) { if(need_obs_thresh) do_obs_thresholding(); @@ -1014,7 +1011,6 @@ void ModeFuzzyEngine::do_obs_merging(const char *default_config, << "inconsistent array dims\n\n"; exit(1); } - if(conf_info.Obs->merge_flag == MergeType_Both || conf_info.Obs->merge_flag == MergeType_Thresh) @@ -1022,7 +1018,7 @@ void ModeFuzzyEngine::do_obs_merging(const char *default_config, if(conf_info.Obs->merge_flag == MergeType_Both || conf_info.Obs->merge_flag == MergeType_Engine) - do_obs_merge_engine(default_config, merge_config); + do_obs_merge_engine("", ""); // // Done @@ -1724,8 +1720,8 @@ void ModeFuzzyEngine::do_fcst_merge_engine(const char *default_config, fcst_engine->ctable = ctable; if(default_config && merge_config) { fcst_engine->conf_info.read_config(default_config, merge_config); - fcst_engine->conf_info.process_config(conf_info.Fcst->var_info->file_type(), - conf_info.Obs->var_info->file_type()); + fcst_engine->conf_info.process_config_traditional(conf_info.Fcst->var_info->file_type(), + conf_info.Obs->var_info->file_type()); path = replace_path(fcst_engine->conf_info.object_pi.color_table.c_str()); fcst_engine->ctable.read(path.c_str()); } @@ -1891,8 +1887,8 @@ void ModeFuzzyEngine::do_obs_merge_engine(const char *default_config, obs_engine->ctable = ctable; if(default_config && merge_config) { obs_engine->conf_info.read_config(default_config, merge_config); - obs_engine->conf_info.process_config(conf_info.Fcst->var_info->file_type(), - conf_info.Obs->var_info->file_type()); + obs_engine->conf_info.process_config_traditional(conf_info.Fcst->var_info->file_type(), + conf_info.Obs->var_info->file_type()); path = replace_path(obs_engine->conf_info.object_pi.color_table.c_str()); obs_engine->ctable.read(path.c_str()); } diff --git a/src/libcode/vx_shapedata/engine.h b/src/libcode/vx_shapedata/engine.h index 8bc5995dfb..50d67628cf 100644 --- a/src/libcode/vx_shapedata/engine.h +++ b/src/libcode/vx_shapedata/engine.h @@ -215,16 +215,19 @@ class ModeFuzzyEngine { void do_fcst_merging(); void do_obs_merging(); + // traditional version void do_fcst_merging(const char *default_config, const char *merge_config); - void do_fcst_merging(const char *default_config, - const char *merge_config, - const ShapeData &merge_data); + + // multivar version + void do_fcst_merging(const ShapeData &merge_data); + + // traditional version void do_obs_merging(const char *default_config, const char *merge_config); - void do_obs_merging(const char *default_config, - const char *merge_config, - const ShapeData &merge_data); + + // multivar version + void do_obs_merging(const ShapeData &merge_data); void do_matching(); diff --git a/src/libcode/vx_shapedata/mode_conf_info.cc b/src/libcode/vx_shapedata/mode_conf_info.cc index a2ebee3c5b..9008152e6f 100644 --- a/src/libcode/vx_shapedata/mode_conf_info.cc +++ b/src/libcode/vx_shapedata/mode_conf_info.cc @@ -17,6 +17,7 @@ using namespace std; #include #include #include +#include #include "mode_conf_info.h" #include "configobjecttype_to_string.h" @@ -51,6 +52,16 @@ ModeConfInfo::~ModeConfInfo() clear(); } +ModeConfInfo & ModeConfInfo::operator=(const ModeConfInfo &s) +{ + if(this == &s) return(*this); + + clear(); + assign(s); + + return(*this); +} + //////////////////////////////////////////////////////////////////////// void ModeConfInfo::init_from_scratch() @@ -77,6 +88,99 @@ void ModeConfInfo::init_from_scratch() //////////////////////////////////////////////////////////////////////// +void ModeConfInfo::assign( const ModeConfInfo &m) +{ + Field_Index_f = m.Field_Index_f; + Field_Index_o = m.Field_Index_o; + N_fields_f = m.N_fields_f; + N_fields_o = m.N_fields_o; + fcst_multivar_logic = m.fcst_multivar_logic; + obs_multivar_logic = m.obs_multivar_logic; + fcst_multivar_compare_index = m.fcst_multivar_compare_index; + obs_multivar_compare_index = m.obs_multivar_compare_index; + fcst_multivar_name = m.fcst_multivar_name; + fcst_multivar_level = m.fcst_multivar_level; + obs_multivar_name = m.obs_multivar_name; + obs_multivar_level = m.obs_multivar_level; + data_type = m.data_type; + conf = m.conf; + centroid_dist_wt = m.centroid_dist_wt; + boundary_dist_wt = m.boundary_dist_wt; + convex_hull_dist_wt = m.convex_hull_dist_wt; + angle_diff_wt = m.angle_diff_wt; + aspect_diff_wt = m.aspect_diff_wt; + area_ratio_wt = m.area_ratio_wt; + int_area_ratio_wt = m.int_area_ratio_wt; + curvature_ratio_wt = m.curvature_ratio_wt; + complexity_ratio_wt = m.complexity_ratio_wt; + inten_perc_ratio_wt = m.inten_perc_ratio_wt; + + fcst_array = 0; + Fcst = 0; + if (N_fields_f > 0) { + fcst_array = new Mode_Field_Info[N_fields_f]; + for (int i=0; i 0) { + obs_array = new Mode_Field_Info[N_fields_o]; + for (int i=0; ilookup_int(conf_key_shift_right); - read_fields (fcst_array, fcst_dict, ftype, 'F'); // the order is important here - read_fields ( obs_array, obs_dict, otype, 'O'); // the order is important here - Fcst = fcst_array; // + 0 - Obs = obs_array; // + 0 + if (field_index == 0) { + read_fields_0 (fcst_array, fcst_dict, ftype, 'F'); + read_fields_1 (fcst_array, fcst_dict, ftype, 'F', 0); + read_fields_0 (obs_array, obs_dict, otype, 'O'); + read_fields_1 (obs_array, obs_dict, otype, 'O', 0); + fcst_array->raw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_fcst_raw_plot)); + obs_array->raw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_obs_raw_plot)); + + } else { + read_fields_1 (fcst_array, fcst_dict, ftype, 'F', field_index); + read_fields_1 (obs_array, obs_dict, otype, 'O', field_index); + } + + Fcst = fcst_array + field_index; + Obs = obs_array + field_index; // Dump the contents of the VarInfo objects if(mlog.verbosity_level() >= 5) { - for (j=0; jdump(cout); - } - for (j=0; jdump(cout); - } // for j - } - for (j=0; jdump(cout); + mlog << Debug(5) + << "Parsed observation field:\n"; + obs_array[field_index].var_info->dump(cout); } - for (j=0; j " - << "fcst and obs convolution radius arrays need to be the same size for traditional mode\n\n"; - exit ( 1 ); - } - if ( fcst_array[j].conv_thresh_array.n_elements() != obs_array[j].conv_thresh_array.n_elements() ) { + if ( fcst_array[field_index].conv_radius_array.n_elements() != + obs_array[field_index].conv_radius_array.n_elements() ) { + mlog << Error << "\nModeConfInfo::process_config_both() -> " + << "fcst and obs convolution radius arrays need to be the same size for traditional mode\n\n"; + exit ( 1 ); + } + if ( fcst_array[field_index].conv_thresh_array.n_elements() != + obs_array[field_index].conv_thresh_array.n_elements() ) { - mlog << Error << "\nModeConfInfo::process_config_both() -> " - << "fcst and obs convolution threshold arrays need to be the same size\n\n"; - exit ( 1 ); - } + mlog << Error << "\nModeConfInfo::process_config_both() -> " + << "fcst and obs convolution threshold arrays need to be the same size\n\n"; + exit ( 1 ); } } - // Conf: fcst_raw_plot - - fcst_array->raw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_fcst_raw_plot)); - - // Conf: obs_raw_plot - - obs_array->raw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_obs_raw_plot)); - - return; + return; } //////////////////////////////////////////////////////////////////////// -void ModeConfInfo::process_config_fcst(GrdFileType ftype) +void ModeConfInfo::process_config_fcst(GrdFileType ftype, int field_index) { int j, k, n; @@ -501,25 +613,26 @@ void ModeConfInfo::process_config_fcst(GrdFileType ftype) fcst_dict = conf.lookup_dictionary(conf_key_fcst); shift_right = fcst_dict->lookup_int(conf_key_shift_right); - read_fields (fcst_array, fcst_dict, ftype, 'F'); // the order is important here - Fcst = fcst_array; // + 0 + if (field_index == 0) { + read_fields_0 (fcst_array, fcst_dict, ftype, 'F'); + read_fields_1 (fcst_array, fcst_dict, ftype, 'F', 0); + fcst_array->raw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_fcst_raw_plot)); + } else { + read_fields_1 (fcst_array, fcst_dict, ftype, 'F', field_index); + } + + Fcst = fcst_array + field_index; - // Dump the contents of the VarInfo objects + // Dump the contents of the VarInfo object if(mlog.verbosity_level() >= 5) { - for (j=0; jdump(cout); - } // for j + mlog << Debug(5) + << "Parsed forecast field:\n"; + fcst_array[field_index].var_info->dump(cout); } - for (j=0; jraw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_fcst_raw_plot)); + evaluate_fcst_settings(field_index); return; @@ -528,7 +641,7 @@ void ModeConfInfo::process_config_fcst(GrdFileType ftype) //////////////////////////////////////////////////////////////////////// -void ModeConfInfo::process_config_obs(GrdFileType otype) +void ModeConfInfo::process_config_obs(GrdFileType otype, int field_index) { int j, k, n; @@ -537,25 +650,25 @@ void ModeConfInfo::process_config_obs(GrdFileType otype) obs_dict = conf.lookup_dictionary(conf_key_obs); shift_right = obs_dict->lookup_int(conf_key_shift_right); - read_fields ( obs_array, obs_dict, otype, 'O'); // the order is important here - Obs = obs_array; // + 0 + if (field_index == 0) { + read_fields_0 (obs_array, obs_dict, otype, 'O'); + read_fields_1 (obs_array, obs_dict, otype, 'O', 0); + obs_array->raw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_obs_raw_plot)); + } else { + read_fields_1 (obs_array, obs_dict, otype, 'O', field_index); + } + Obs = obs_array + field_index; - // Dump the contents of the VarInfo objects + // Dump the contents of the VarInfo object if(mlog.verbosity_level() >= 5) { - for (j=0; jdump(cout); - } // for j + mlog << Debug(5) + << "Parsed observation field:\n"; + obs_array[field_index].var_info->dump(cout); } - for (j=0; jraw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_obs_raw_plot)); + evaluate_obs_settings(field_index); return; @@ -563,6 +676,140 @@ void ModeConfInfo::process_config_obs(GrdFileType otype) //////////////////////////////////////////////////////////////////////// +void ModeConfInfo::config_set_all_percentile_thresholds(const std::vector &fdata, + const std::vector &odata) +{ + + // for each forecast input, it's either not a percentile threshold, + // is a simple (single input) percentile, is a frequency bias percentile threshold, or is a + // climatology percentile (we don't have climatology). + // + // As a complication we have both conv_thresh and merge_thresh, so what if they are different? + // right now I simply go with frequency bias as highest priority + + // indices of forecast and obs inputs that have frequency bias percentile thresholding + vector fcst_freq, obs_freq; + + // indices (common to forecast and obs) that require both inputs (fcst and obs) + // which is either frequency bias, or sample obs with a forecast input, or sample fcst + // with an obs input + vector indices_with_both; + + for (int j=0; j= N_fields_o) { + mlog << Error << "\nModeConfInfo::config_set_all_percentile_thresholds\n" + << " SOP Thresholding on fcst index " << j+1 + << " out of range of obs " << N_fields_o + 1 << "\n\n"; + exit ( 1 ); + } + // deal with this later + indices_with_both.push_back(j); + break; + case perc_thresh_freq_bias: + // currently expect matching index in obs and fcst, so check to make sure not both + if (j >= N_fields_o) { + mlog << Error << "\nModeConfInfo::config_set_all_percentile_thresholds\n" + << " FBIAS Thresholding on fcst index " << j+1 + << " out of range of obs " << N_fields_o + 1 << "\n\n"; + exit ( 1 ); + } + fcst_freq.push_back(j); + indices_with_both.push_back(j); + break; + default: + break; + } + } + for (int j=0; j= N_fields_f) { + mlog << Error << "\nModeConfInfo::config_set_all_percentile_thresholds\n" + << " SFP Thresholding on obs index " << j+1 + << " out of range of fcst " << N_fields_f + 1 << "\n\n"; + exit ( 1 ); + } + // deal with this later + if (find(indices_with_both.begin(), indices_with_both.end(), j) == indices_with_both.end()) { + indices_with_both.push_back(j); + } + break; + case perc_thresh_freq_bias: + // currently expect matching index in obs and fcst, so check to make sure not both + if (j >= N_fields_f) { + mlog << Error << "\nModeConfInfo::config_set_all_percentile_thresholds\n" + << " FBIAS Thresholding on obs index " << j+1 + << " out of range of fcst " << N_fields_f + 1 << "\n\n"; + exit ( 1 ); + } + // deal with this later + obs_freq.push_back(j); + if (find(indices_with_both.begin(), indices_with_both.end(), j) == indices_with_both.end()) { + indices_with_both.push_back(j); + } + break; + default: + break; + } + } + + // make sure no overlap in fcst/obs frequencies, as that's a no no + for (size_t i=0; iis_array() ) { exit ( 1 ); } + } - info_array[j].set(true, j, e->dict_value(), &conf, type, _fo, false); + return; - if ( j == 0 ) { +} // if is array - for (k=1; klookup(conf_key_field); + if ( !ee ) { + + mlog << "\nModeConfInfo::read_fields () -> \"field\" entry not found in dictionary!\n\n"; + + exit ( 1 ); + +} + +const Dictionary * field = ee->dict(); + +const int N = ( (field->is_array()) ? (field->n_entries()) : 1 ); + +if ( field->is_array() ) { + + const DictionaryEntry * e = 0; + const Dictionary & D = *field; + + e = D[field_index]; + if ( (e->type() != DictionaryType) && (e->type() != ArrayType) ) { + mlog << Error + << "\nModeConfInfo::read_fields() -> field entry # " << field_index+1 << " is not a dictionary!\n\n"; + exit ( 1 ); + + } + + info_array[field_index].set(true, field_index, e->dict_value(), &conf, type, _fo, false); return; } // if is array - // // nope, traditional mode // -N_fields_f = N; -N_fields_o = N; + if (field_index != 0) { + mlog << Error + << "\nModeConfInfo::read_fields() -> traditional mode but looking for a field array index " << field_index + << "\n\n"; + exit ( 1 ); + } + + info_array[0].set(false, 0, dict, &conf, type, _fo); // @@ -1287,6 +1582,46 @@ void ModeConfInfo::set_data_type(ModeDataType type) //////////////////////////////////////////////////////////////////////// +GrdFileType ModeConfInfo::file_type_for_field(bool isFcst, int field_index) +{ + // look at the dictionary for the obs or forecast at index, with + // parents + + Dictionary * dict = (Dictionary *) 0; + if (isFcst) { + dict = conf.lookup_dictionary(conf_key_fcst); + } else { + dict = conf.lookup_dictionary(conf_key_obs); + } + + const DictionaryEntry *e = dict->lookup(conf_key_field); + if ( !e ) { + mlog << "\nModeConfInfo::read_fields () -> \"field\" entry not found in dictionary!\n\n"; + exit ( 1 ); + } + + const Dictionary * field = e->dict(); + const int N = ( (field->is_array()) ? (field->n_entries()) : 1 ); + if (field_index < 0 || field_index >= N) { + mlog << Error + << "\nModeConfInfo::file_type_for_field() -> " + << "index " << field_index+1 << " out of range 0 to " << N + << "\n\n"; + exit (1); + } + e = (*field)[field_index]; + if ( (e->type() != DictionaryType) && (e->type() != ArrayType) ) { + mlog << Error + << "\nModeConfInfo::file_type_for_field() -> " + << "field entry # " << field_index+1 << " is not a dictionary!\n\n"; + exit ( 1 ); + } + Dictionary *dictf = e->dict_value(); + return parse_conf_file_type(dictf); +} + +//////////////////////////////////////////////////////////////////////// + int ModeConfInfo::n_runs() const @@ -1406,7 +1741,12 @@ void ModeConfInfo::check_multivar_not_implemented() << "\nModeConfInfo::check_multivar_not_implemented():\n" << " merge_flag ENGINE or BOTH not implemented for multivariate mode\n\n"; status = true; - break; + } + if (fcst_array[i].conv_thresh_array.n() > 1 || fcst_array[i].merge_thresh_array.n() > 1) { + mlog << Error + << "\nModeConfInfo::check_multivar_not_implemented():\n" + << " more than one conv_thresh or merge_thresh per input is not allowed in multivariate mode\n\n"; + status = true; } } } @@ -1419,6 +1759,12 @@ void ModeConfInfo::check_multivar_not_implemented() status = true; break; } + if (obs_array[i].conv_thresh_array.n() > 1 || obs_array[i].merge_thresh_array.n() > 1) { + mlog << Error + << "\nModeConfInfo::check_multivar_not_implemented():\n" + << " more than one conv_thresh or merge_thresh per input is not allowed in multivariate mode\n\n"; + status = true; + } } } @@ -1433,52 +1779,43 @@ void ModeConfInfo::check_multivar_not_implemented() //////////////////////////////////////////////////////////////////////// -void ModeConfInfo::check_multivar_perc_thresh(bool isSimple, bool isSimpleMerge) const +PercThreshType ModeConfInfo::perctype(const Mode_Field_Info &f) const { - if (isSimple) { - if (data_type == ModeDataType_MvMode_Fcst) { - for (int j=0; j< Fcst->conv_thresh_array.n(); ++j) { - if (Fcst->conv_thresh_array[j].get_ptype() == perc_thresh_sample_obs) { - mlog << Warning - << "\nModeConfInfo::check_multivar_perc_thresh:\n" - << " Thresholding with 'SOP' in a forecast input not well defined for multivariate mode simple object creation\n" - << " 'SFP' will be used as a replacement\n\n"; - } - } - } else if (data_type == ModeDataType_MvMode_Obs) { - for (int j=0; j< Obs->conv_thresh_array.n(); ++j) { - if (Obs->conv_thresh_array[j].get_ptype() == perc_thresh_sample_fcst) { - mlog << Warning - << "\nModeConfInfo::check_multivar_perc_thresh:\n" - << " Thresholding with 'SFP' in an obs input not well defined for multivariate mode simple object creation\n" - << " 'SOP' will be used as a replacement\n\n"; - } - } - } + PercThreshType pm=no_perc_thresh_type;; + PercThreshType pc=no_perc_thresh_type;; + if (f.merge_thresh_array.n() > 0) { + pm = f.merge_thresh_array[0].get_ptype(); } - - if (isSimpleMerge) { - if (data_type == ModeDataType_MvMode_Fcst) { - for (int j=0; j< Fcst->merge_thresh_array.n(); ++j) { - if (Fcst->merge_thresh_array[j].get_ptype() == perc_thresh_sample_obs) { - mlog << Warning - << "\nModeConfInfo::check_multivar_perc_thresh:\n" - << " Thresholding with 'SOP' in a forecast input not well defined for multivariate mode simple object creation\n" - << " 'SFP' will be used as a replacement\n\n"; - } - } - } else if (data_type == ModeDataType_MvMode_Obs) { - for (int j=0; j< Obs->merge_thresh_array.n(); ++j) { - if (Obs->merge_thresh_array[j].get_ptype() == perc_thresh_sample_fcst) { - mlog << Warning - << "\nModeConfInfo::check_multivar_perc_thresh():\n" - << " Thresholding with 'SFP' in an obs input not well defined for multivariate mode simple object creation\n" - << " 'SOP' will be used as a replacement\n\n"; - } - } - } + if (f.conv_thresh_array.n() > 0) { + pc = f.conv_thresh_array[0].get_ptype(); } -} + if (pm == perc_thresh_sample_climo || pc == perc_thresh_sample_climo) { + mlog << Error << "\nModeConfInfo::perctype()\n" + << " Thresholding with 'SCP' in an input not implemented for multivariate mode\n\n"; + exit ( 1 ); + } + if (pm == perc_thresh_climo_dist || pc == perc_thresh_climo_dist) { + mlog << Error << "\nModeConfInfo::perctype()\n" + << " Thresholding with 'CDP' in an input not implemented for multivariate mode\n\n"; + exit ( 1 ); + } + if (pm == perc_thresh_freq_bias || + pc == perc_thresh_freq_bias) { + return perc_thresh_freq_bias; + } + // put in tests if they are not the same and + // both are sample settings, so the right one + // gets returned (pass in fcst/obs boolean) + if (pm == perc_thresh_sample_obs || + pc == perc_thresh_sample_obs) { + return perc_thresh_sample_obs; + } + if (pm == perc_thresh_sample_fcst || + pc == perc_thresh_sample_fcst) { + return perc_thresh_sample_fcst; + } + return no_perc_thresh_type; +} //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_shapedata/mode_conf_info.h b/src/libcode/vx_shapedata/mode_conf_info.h index 7fab368dc4..fcaca3e5a0 100644 --- a/src/libcode/vx_shapedata/mode_conf_info.h +++ b/src/libcode/vx_shapedata/mode_conf_info.h @@ -25,6 +25,7 @@ #include "mode_field_info.h" #include "mode_data_type.h" +#include "mode_input_data.h" //////////////////////////////////////////////////////////////////////// @@ -74,12 +75,17 @@ class ModeConfInfo { int N_fields_f; // for traditional mode expect N_fields_f/_o to be the same int N_fields_o; + void assign(const ModeConfInfo &); + public: ModeConfInfo(); ~ModeConfInfo(); + // attempt to implement this + ModeConfInfo & operator=(const ModeConfInfo &); + void clear(); // sets both obs and fcst field indices if traditional mode, or mvmode 'both' @@ -106,7 +112,15 @@ class ModeConfInfo { void check_multivar_not_implemented(); - void check_multivar_perc_thresh(bool isSimple, bool isSimpleMerge) const; + // check all inputs for climatology percentiles, which are not implemented + // warn about forecasts and obs set backwards + // void check_multivar_perc_thresh(const Mode_Field_Info &f, bool isFcst) const; + + + // // return index to other input if the particular input is a frequency bias percentile + // // return -1 if input is not a fequency bias percentile + // // if input is forecast, 'other' is obs, and vice versa + // int frequency_bias_other_index(bool input_is_fcst, int input_index) const; ///////////////////////////////////////////////////////////////////// @@ -122,7 +136,7 @@ class ModeConfInfo { // and to distinguish traditional mode from mvmode ModeDataType data_type; - + ///////////////////////////////////////////////////////////////////// @@ -135,9 +149,21 @@ class ModeConfInfo { void read_config (const char * default_filename, const char * user_filename); - void process_config (GrdFileType ftype, GrdFileType otype, ModeDataType dt=ModeDataType_Traditional); + void process_config_traditional(GrdFileType ftype, GrdFileType otype); - void read_fields (Mode_Field_Info * &, Dictionary * dict, GrdFileType, char _fo); + void process_config_except_fields(); + + void process_config_field (GrdFileType ftype, GrdFileType otype, ModeDataType dt, int field_index); + + void config_set_all_percentile_thresholds(const std::vector &fdata, + const std::vector &odata); + PercThreshType perctype(const Mode_Field_Info &f) const; + + // deal with zeroth field + void read_fields_0 (Mode_Field_Info * &, Dictionary * dict, GrdFileType, char _fo); + + // deal with non-zeroth field + void read_fields_1 (Mode_Field_Info * &, Dictionary * dict, GrdFileType, char _fo, int field_index); PiecewiseLinear * parse_interest_function(Dictionary * dict, const char * conf_key_if); @@ -269,12 +295,14 @@ class ModeConfInfo { void set_data_type(ModeDataType type); + GrdFileType file_type_for_field(bool isFcst, int field_index); + private: // some private methods - void process_config_both(GrdFileType ftype, GrdFileType otype); - void process_config_fcst(GrdFileType ftype); - void process_config_obs(GrdFileType otype); + void process_config_both(GrdFileType ftype, GrdFileType otype, int field_index=0); + void process_config_fcst(GrdFileType ftype, int field_index=0); + void process_config_obs(GrdFileType otype, int field_index=0); void evaluate_fcst_settings(int); void evaluate_obs_settings(int); diff --git a/src/libcode/vx_shapedata/mode_field_info.cc b/src/libcode/vx_shapedata/mode_field_info.cc index 8dd024e803..fcd652a09a 100644 --- a/src/libcode/vx_shapedata/mode_field_info.cc +++ b/src/libcode/vx_shapedata/mode_field_info.cc @@ -85,6 +85,32 @@ return ( * this ); } +//////////////////////////////////////////////////////////////////////// + +void Mode_Field_Info::clone(const Mode_Field_Info & i) +{ + dict = i.dict; + conf = i.conf; + gft = i.gft; + FO = i.FO; + Multivar = i.Multivar; + index = i.index; + conv_radius = i.conv_radius; + vld_thresh = i.vld_thresh; + + var_info = i.var_info->clone(); + + conv_radius_array = i.conv_radius_array; + conv_thresh_array = i.conv_thresh_array; + merge_thresh_array = i.merge_thresh_array; + conv_thresh = i.conv_thresh; + merge_thresh = i.merge_thresh; + merge_flag = i.merge_flag; + raw_pi = i.raw_pi; + filter_attr_map = i.filter_attr_map; + file_type = i.file_type; +} + //////////////////////////////////////////////////////////////////////// @@ -143,6 +169,8 @@ merge_thresh.clear(); merge_flag = MergeType_Engine; +file_type = FileType_None; + // raw_pi.clear(); } @@ -310,8 +338,8 @@ if ( FO == 'F' ) raw_pi = parse_conf_plot_info(conf->lookup_dictionary(conf_key else raw_pi = parse_conf_plot_info(conf->lookup_dictionary(conf_key_obs_raw_plot)); - - +file_type = parse_conf_file_type(dict); + // // done // diff --git a/src/libcode/vx_shapedata/mode_field_info.h b/src/libcode/vx_shapedata/mode_field_info.h index 132f2df4b2..bb7e08cb57 100644 --- a/src/libcode/vx_shapedata/mode_field_info.h +++ b/src/libcode/vx_shapedata/mode_field_info.h @@ -42,10 +42,9 @@ class Mode_Field_Info { void assign(const Mode_Field_Info &); + Dictionary * dict; // not allocated, based on reading the config file - Dictionary * dict; // not allocated - - MetConfig * conf; // not allocated + MetConfig * conf; // not allocated, based on reading the config file GrdFileType gft; @@ -59,6 +58,7 @@ class Mode_Field_Info { ~Mode_Field_Info(); Mode_Field_Info(const Mode_Field_Info &); Mode_Field_Info & operator=(const Mode_Field_Info &); + void clone(const Mode_Field_Info &); void clear(); @@ -85,6 +85,8 @@ class Mode_Field_Info { PlotInfo raw_pi; // Raw forecast plotting info + GrdFileType file_type; // each field can have a configured file type + // // member functions // diff --git a/src/libcode/vx_shapedata/mode_input_data.h b/src/libcode/vx_shapedata/mode_input_data.h new file mode 100644 index 0000000000..0aa9b4c36d --- /dev/null +++ b/src/libcode/vx_shapedata/mode_input_data.h @@ -0,0 +1,42 @@ +// ** Copyright UCAR (c) 1992 - 2023 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + + +#ifndef __MODE_INPUT_DATA_H__ +#define __MODE_INPUT_DATA_H__ + +#include +#include "data_plane.h" +#include "grid_base.h" + +//////////////////////////////////////////////////////////////////////// + + +class ModeInputData { + + private: + + public: + + ModeInputData(const std::string &name, const DataPlane &dp, const Grid &g) : + _name(name), _dataPlane(dp), _grid(g) {} + + ~ModeInputData() {} + + std::string _name; + DataPlane _dataPlane; + Grid _grid; +}; + + +#endif + + +///////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/mode/Makefile.am b/src/tools/core/mode/Makefile.am index d8dc507385..34bb6cc8dd 100644 --- a/src/tools/core/mode/Makefile.am +++ b/src/tools/core/mode/Makefile.am @@ -13,6 +13,7 @@ include ${top_srcdir}/Make-include bin_PROGRAMS = mode mode_SOURCES = mode_usage.cc \ mode_frontend.cc \ + mode_superobject.cc \ multivar_data.cc \ multivar_frontend.cc \ mode.cc \ diff --git a/src/tools/core/mode/Makefile.in b/src/tools/core/mode/Makefile.in index 741c5fea4f..10f09a3200 100644 --- a/src/tools/core/mode/Makefile.in +++ b/src/tools/core/mode/Makefile.in @@ -102,9 +102,9 @@ CONFIG_CLEAN_VPATH_FILES = am__installdirs = "$(DESTDIR)$(bindir)" PROGRAMS = $(bin_PROGRAMS) am_mode_OBJECTS = mode-mode_usage.$(OBJEXT) \ - mode-mode_frontend.$(OBJEXT) mode-multivar_data.$(OBJEXT) \ - mode-multivar_frontend.$(OBJEXT) mode-mode.$(OBJEXT) \ - mode-combine_boolplanes.$(OBJEXT) \ + mode-mode_frontend.$(OBJEXT) mode-mode_superobject.$(OBJEXT) \ + mode-multivar_data.$(OBJEXT) mode-multivar_frontend.$(OBJEXT) \ + mode-mode.$(OBJEXT) mode-combine_boolplanes.$(OBJEXT) \ mode-objects_from_netcdf.$(OBJEXT) mode-mode_ps_file.$(OBJEXT) \ mode-plot_engine.$(OBJEXT) mode-page_1.$(OBJEXT) \ mode-fcst_enlarge_page.$(OBJEXT) \ @@ -138,6 +138,7 @@ am__depfiles_remade = ./$(DEPDIR)/mode-cluster_page.Po \ ./$(DEPDIR)/mode-mode_exec.Po \ ./$(DEPDIR)/mode-mode_frontend.Po \ ./$(DEPDIR)/mode-mode_ps_file.Po \ + ./$(DEPDIR)/mode-mode_superobject.Po \ ./$(DEPDIR)/mode-mode_usage.Po \ ./$(DEPDIR)/mode-multivar_data.Po \ ./$(DEPDIR)/mode-multivar_frontend.Po \ @@ -350,6 +351,7 @@ top_srcdir = @top_srcdir@ MAINTAINERCLEANFILES = Makefile.in mode_SOURCES = mode_usage.cc \ mode_frontend.cc \ + mode_superobject.cc \ multivar_data.cc \ multivar_frontend.cc \ mode.cc \ @@ -503,6 +505,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-mode_exec.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-mode_frontend.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-mode_ps_file.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-mode_superobject.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-mode_usage.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-multivar_data.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-multivar_frontend.Po@am__quote@ # am--include-marker @@ -560,6 +563,20 @@ mode-mode_frontend.obj: mode_frontend.cc @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(mode_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o mode-mode_frontend.obj `if test -f 'mode_frontend.cc'; then $(CYGPATH_W) 'mode_frontend.cc'; else $(CYGPATH_W) '$(srcdir)/mode_frontend.cc'; fi` +mode-mode_superobject.o: mode_superobject.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(mode_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT mode-mode_superobject.o -MD -MP -MF $(DEPDIR)/mode-mode_superobject.Tpo -c -o mode-mode_superobject.o `test -f 'mode_superobject.cc' || echo '$(srcdir)/'`mode_superobject.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/mode-mode_superobject.Tpo $(DEPDIR)/mode-mode_superobject.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mode_superobject.cc' object='mode-mode_superobject.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(mode_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o mode-mode_superobject.o `test -f 'mode_superobject.cc' || echo '$(srcdir)/'`mode_superobject.cc + +mode-mode_superobject.obj: mode_superobject.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(mode_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT mode-mode_superobject.obj -MD -MP -MF $(DEPDIR)/mode-mode_superobject.Tpo -c -o mode-mode_superobject.obj `if test -f 'mode_superobject.cc'; then $(CYGPATH_W) 'mode_superobject.cc'; else $(CYGPATH_W) '$(srcdir)/mode_superobject.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/mode-mode_superobject.Tpo $(DEPDIR)/mode-mode_superobject.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mode_superobject.cc' object='mode-mode_superobject.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(mode_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o mode-mode_superobject.obj `if test -f 'mode_superobject.cc'; then $(CYGPATH_W) 'mode_superobject.cc'; else $(CYGPATH_W) '$(srcdir)/mode_superobject.cc'; fi` + mode-multivar_data.o: multivar_data.cc @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(mode_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT mode-multivar_data.o -MD -MP -MF $(DEPDIR)/mode-multivar_data.Tpo -c -o mode-multivar_data.o `test -f 'multivar_data.cc' || echo '$(srcdir)/'`multivar_data.cc @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/mode-multivar_data.Tpo $(DEPDIR)/mode-multivar_data.Po @@ -876,6 +893,7 @@ distclean: distclean-am -rm -f ./$(DEPDIR)/mode-mode_exec.Po -rm -f ./$(DEPDIR)/mode-mode_frontend.Po -rm -f ./$(DEPDIR)/mode-mode_ps_file.Po + -rm -f ./$(DEPDIR)/mode-mode_superobject.Po -rm -f ./$(DEPDIR)/mode-mode_usage.Po -rm -f ./$(DEPDIR)/mode-multivar_data.Po -rm -f ./$(DEPDIR)/mode-multivar_frontend.Po @@ -936,6 +954,7 @@ maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/mode-mode_exec.Po -rm -f ./$(DEPDIR)/mode-mode_frontend.Po -rm -f ./$(DEPDIR)/mode-mode_ps_file.Po + -rm -f ./$(DEPDIR)/mode-mode_superobject.Po -rm -f ./$(DEPDIR)/mode-mode_usage.Po -rm -f ./$(DEPDIR)/mode-multivar_data.Po -rm -f ./$(DEPDIR)/mode-multivar_frontend.Po diff --git a/src/tools/core/mode/combine_boolplanes.h b/src/tools/core/mode/combine_boolplanes.h index e879827740..084a31aed1 100644 --- a/src/tools/core/mode/combine_boolplanes.h +++ b/src/tools/core/mode/combine_boolplanes.h @@ -20,7 +20,8 @@ #include "two_d_array.h" #include "bool_calc.h" #include "vx_pxm.h" - +#include +#include //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/mode/mode.cc b/src/tools/core/mode/mode.cc index b7277b0110..d71a7698b5 100644 --- a/src/tools/core/mode/mode.cc +++ b/src/tools/core/mode/mode.cc @@ -57,6 +57,7 @@ // 020 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main // 021 06/09/23 Albo Major changes for multivariate mode // 022 11/02/23 Halley Gotway MET #2724 add OpenMP to convolution +// 023 12/27/23 Albo MET #2745 more unit tests, read data one time, percentile thresholding // //////////////////////////////////////////////////////////////////////// @@ -78,6 +79,7 @@ using namespace std; #include "string_array.h" #include "mode_usage.h" #include "mode_frontend.h" +#include "multivar_frontend.h" #include "mode_conf_info.h" #ifdef WITH_PYTHON @@ -97,7 +99,6 @@ using namespace std; /////////////////////////////////////////////////////////////////////// -extern int mode_frontend(const StringArray &); extern int multivar_frontend(const StringArray &); extern const char * const program_name; @@ -186,8 +187,10 @@ int met_main(int argc, char * argv []) config.check_multivar_not_implemented(); // run the multivar version of mode - - status = multivar_frontend(Argv); + + MultivarFrontEnd *frontend = new MultivarFrontEnd(); + status = frontend->run(Argv); + if ( frontend ) { delete frontend; frontend = 0; } } else { diff --git a/src/tools/core/mode/mode_exec.cc b/src/tools/core/mode/mode_exec.cc index f381ff549b..d0c2e01b7a 100644 --- a/src/tools/core/mode/mode_exec.cc +++ b/src/tools/core/mode/mode_exec.cc @@ -35,7 +35,6 @@ static const char * cts_str[n_cts] = {"RAW", "OBJECT"}; static const char program_name [] = "mode"; static const char * default_config_filename = "MET_BASE/config/MODEConfig_default"; -static const char * default_multivar_config_filename = "MET_BASE/config/MODEMultivarConfig_default"; // took this out of the do_conv_thresh() method static int local_r_index = -1; @@ -60,12 +59,11 @@ static void replaceAll(std::string& str, const std::string& from, const std::str /////////////////////////////////////////////////////////////////////// -ModeExecutive::ModeExecutive(Processing_t p) +ModeExecutive::ModeExecutive()//Processing_t p) { init_from_scratch(); - ptype = p; } @@ -132,8 +130,6 @@ void ModeExecutive::clear() R_index = T_index = 0; - ptype = TRADITIONAL; - // // done // @@ -154,7 +150,7 @@ void ModeExecutive::init_traditional(int n_files) R_index = T_index = 0; - conf_read(default_config_filename); + conf_read(); // Get the forecast and observation file types from config, if present ftype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_fcst)); @@ -180,7 +176,7 @@ void ModeExecutive::init_traditional(int n_files) otype = obs_mtddf->file_type(); // Process the configuration - engine.conf_info.process_config(ftype, otype); + engine.conf_info.process_config_traditional(ftype, otype); int nf = engine.conf_info.n_fields_f(); // same as obs for traditional mode if (nf != n_files) { mlog << Error << "\nNumber of input files " << n_files << " Not equal to config number of fields " @@ -206,161 +202,19 @@ void ModeExecutive::init_traditional(int n_files) /////////////////////////////////////////////////////////////////////// - -void ModeExecutive::init_multivar_verif_grid() +void ModeExecutive::init_multivar_simple(int j, int n_files, ModeDataType dtype, + const ModeConfInfo &conf) { - - Met2dDataFileFactory mtddf_factory; - R_index = T_index = 0; - - conf_read(default_multivar_config_filename); - - // Get the forecast and observation file types from config, if present - ftype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_fcst)); - otype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_obs)); - - - // Read observation file - if(!(obs_mtddf = mtddf_factory.new_met_2d_data_file(obs_file.c_str(), otype))) { - mlog << Error << "\nTrouble reading observation file \"" - << obs_file << "\"\n\n"; - exit(1); - } - - // Read forecast file - if(!(fcst_mtddf = mtddf_factory.new_met_2d_data_file(fcst_file.c_str(), ftype))) { - mlog << Error << "\nTrouble reading forecast file \"" - << fcst_file << "\"\n\n"; - exit(1); - } - - // Store the input data file types - ftype = fcst_mtddf->file_type(); - otype = obs_mtddf->file_type(); - - // Process the configuration - engine.conf_info.process_config(ftype, otype, ModeDataType_MvMode_Both); - - // check one again for multivar problems - engine.conf_info.check_multivar_not_implemented(); - - const int shift = engine.conf_info.shift_right; - - fcst_mtddf->set_shift_right(shift); - obs_mtddf->set_shift_right(shift); - - // List the input files - mlog << Debug(1) - << "Forecast File: " << fcst_file << "\n" - << "Observation File: " << obs_file << "\n"; - - engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); - - return; - -} - -/////////////////////////////////////////////////////////////////////// - -void ModeExecutive::init_multivar_simple(int n_files, ModeDataType dtype) - -{ - - Met2dDataFileFactory mtddf_factory; - - R_index = T_index = 0; - - conf_read(default_multivar_config_filename); + engine.conf_info = conf; // tell the engine which type of data it is engine.set_data_type(dtype); - // Get the forecast and observation file types from config, if present - ftype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_fcst)); - otype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_obs)); - - - // Read observation file or forecast file - if (dtype == ModeDataType_MvMode_Obs) - { - if(!(obs_mtddf = mtddf_factory.new_met_2d_data_file(obs_file.c_str(), otype))) { - mlog << Error << "\nTrouble reading observation file \"" - << obs_file << "\"\n\n"; - exit(1); - - } - // Store the input data file types - otype = obs_mtddf->file_type(); - } - else if (dtype == ModeDataType_MvMode_Fcst) - { - // Read forecast file - if(!(fcst_mtddf = mtddf_factory.new_met_2d_data_file(fcst_file.c_str(), ftype))) { - mlog << Error << "\nTrouble reading forecast file \"" - << fcst_file << "\"\n\n"; - exit(1); - } - // Store the input data file types - ftype = fcst_mtddf->file_type(); - } - else { - mlog << Error << "\nModeExecutive::init_multivar_simple()->" - << "simple object creation requires obs or forecast mode, got " - << sprintModeDataType(dtype) << "\n\n"; - exit(1); - } + engine.conf_info.set_field_index(j); - // Process the configuration - engine.conf_info.process_config(ftype, otype, dtype); - int nf; - if (dtype == ModeDataType_MvMode_Obs) { - nf = engine.conf_info.n_fields_o(); - } else if (dtype == ModeDataType_MvMode_Fcst) { - nf = engine.conf_info.n_fields_f(); - } else { - mlog << Error << "\nModeExecutive::init_multivar_simple()->" - << "simple object creation requires obs or forecast mode, got " - << sprintModeDataType(dtype) << "\n\n"; - exit(1); - } - if (nf != n_files) { - mlog << Error << "\nModeExecutive::init_multivar_simple()->" - << "Number of input files " << n_files << " Not equal to config number of fields " - << nf << "\n\n"; - exit(1); - } - - // check one again for multivar problems - engine.conf_info.check_multivar_not_implemented(); - - const int shift = engine.conf_info.shift_right; - - if (dtype != ModeDataType_MvMode_Fcst) { - obs_mtddf->set_shift_right(shift); - } - - if (dtype != ModeDataType_MvMode_Obs) { - fcst_mtddf->set_shift_right(shift); - } - - if (dtype == ModeDataType_MvMode_Obs) { - mlog << Debug(1) - << "Observation File: " << obs_file << "\n"; - - } else if (dtype == ModeDataType_MvMode_Fcst) { - mlog << Debug(1) - << "Forecast File: " << fcst_file << "\n"; - } - else { - // List the input files - mlog << Debug(1) - << "Forecast File: " << fcst_file << "\n" - << "Observation File: " << obs_file << "\n"; - } - - engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); + // engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); return; @@ -368,34 +222,21 @@ void ModeExecutive::init_multivar_simple(int n_files, ModeDataType dtype) /////////////////////////////////////////////////////////////////////// -void ModeExecutive::init_multivar_intensities(GrdFileType ftype, GrdFileType otype) +void ModeExecutive::init_multivar_intensities(const ModeConfInfo &conf) { R_index = T_index = 0; - conf_read(default_multivar_config_filename); - - // Get the forecast and observation file types from config, if present - GrdFileType l_ftype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_fcst)); - GrdFileType l_otype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_obs)); - - // actually use the values passed in instead, the config ones are not set for this second pass - l_ftype = ftype; - l_otype = otype; - - // Process the configuration - engine.conf_info.process_config(l_ftype, l_otype, ModeDataType_MvMode_Both); + engine.conf_info = conf; + + // tell the engine which type of data it is + engine.set_data_type(ModeDataType_MvMode_Both); // check one again for multivar problems engine.conf_info.check_multivar_not_implemented(); - // NOTE: do not do shifting, should have been done in the first pass - // const int shift = engine.conf_info.shift_right; - // fcst_mtddf->set_shift_right(shift); - // obs_mtddf->set_shift_right(shift); - - engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); + // engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); return; @@ -404,16 +245,8 @@ void ModeExecutive::init_multivar_intensities(GrdFileType ftype, GrdFileType oty /////////////////////////////////////////////////////////////////////// -void ModeExecutive::check_multivar_perc_thresh_settings() -{ - engine.conf_info.check_multivar_perc_thresh(ptype == ModeExecutive::MULTIVAR_SIMPLE, - ptype == ModeExecutive::MULTIVAR_SIMPLE_MERGE); -} -/////////////////////////////////////////////////////////////////////// - - -void ModeExecutive::setup_fcst_obs_data_traditional() +void ModeExecutive::setup_traditional_fcst_obs_data() { @@ -427,7 +260,7 @@ void ModeExecutive::setup_fcst_obs_data_traditional() if ( !(fcst_mtddf->data_plane(*(engine.conf_info.Fcst->var_info), Fcst_sd.data)) ) { - mlog << Error << "\nModeExecutive::setup_fcst_obs_data_traditional() -> " + mlog << Error << "\nModeExecutive::setup_traditionalfcst_obs_data() -> " << "can't get forecast data \"" << engine.conf_info.Fcst->var_info->magic_str() << "\" from file \"" << fcst_file << "\"\n\n"; @@ -438,7 +271,7 @@ void ModeExecutive::setup_fcst_obs_data_traditional() if ( !(obs_mtddf->data_plane(*(engine.conf_info.Obs->var_info), Obs_sd.data)) ) { - mlog << Error << "\nModeExecutive::setup_fcst_obs_data_traditional() -> " + mlog << Error << "\nModeExecutive::setup_traditional_fcst_obs_data() -> " << "can't get observation data \"" << engine.conf_info.Obs->var_info->magic_str() << "\" from file \"" << obs_file << "\"\n\n"; @@ -577,68 +410,39 @@ void ModeExecutive::setup_fcst_obs_data_traditional() /////////////////////////////////////////////////////////////////////// - -void ModeExecutive::setup_verification_grid() - +void ModeExecutive::setup_verification_grid(const ModeInputData &fcst, + const ModeInputData &obs, + const ModeConfInfo &conf) { - - // ShapeData fcst_sd, obs_sd; - double fmin, omin, fmax, omax; + R_index = T_index = 0; + engine.conf_info = conf; + engine.conf_info.check_multivar_not_implemented(); Fcst_sd.clear(); Obs_sd.clear(); - - // Read the gridded data from the input forecast file - - if ( !(fcst_mtddf->data_plane(*(engine.conf_info.Fcst->var_info), Fcst_sd.data)) ) { - - mlog << Error << "\nModeExecutive::setup_verification_grid() -> " - << "can't get forecast data \"" - << engine.conf_info.Fcst->var_info->magic_str() - << "\" from file \"" << fcst_file << "\"\n\n"; - exit(1); - } - - // Read the gridded data from the input observation file - - if ( !(obs_mtddf->data_plane(*(engine.conf_info.Obs->var_info), Obs_sd.data)) ) { - - mlog << Error << "\nModeExecutive::setup_verification_grid() -> " - << "can't get observation data \"" - << engine.conf_info.Obs->var_info->magic_str() - << "\" from file \"" << obs_file << "\"\n\n"; - exit(1); - } - - // Determine the verification grid - + Fcst_sd.data = fcst._dataPlane; + Obs_sd.data = obs._dataPlane; + + // set this local conf to point to forecast 0, so that that regrid info + // can be accessed and it can be decided if we are to use fcst or obs + // input + engine.conf_info.set_data_type(ModeDataType_MvMode_Fcst); + engine.conf_info.set_field_index(0); grid = parse_vx_grid(engine.conf_info.Fcst->var_info->regrid(), - &(fcst_mtddf->grid()), &(obs_mtddf->grid())); - + &fcst._grid, &obs._grid); return; } /////////////////////////////////////////////////////////////////////// - -void ModeExecutive::setup_fcst_data(const Grid &verification_grid) - +void ModeExecutive::setup_multivar_fcst_data(const Grid &verification_grid, + const ModeInputData &input) { - // ShapeData fcst_sd, obs_sd; double fmin, fmax; Fcst_sd.clear(); - - // Read the gridded data from the input forecast file - if ( !(fcst_mtddf->data_plane(*(engine.conf_info.Fcst->var_info), Fcst_sd.data)) ) { - - mlog << Error << "\nModeExecutive::setup_fcst_data() -> " - << "can't get forecast data \"" - << engine.conf_info.Fcst->var_info->magic_str() - << "\" from file \"" << fcst_file << "\"\n\n"; - exit(1); - } + Fcst_sd.data = input._dataPlane; grid = verification_grid; @@ -647,12 +451,11 @@ void ModeExecutive::setup_fcst_data(const Grid &verification_grid) engine.set_grid(&grid); // Regrid, if necessary - - if ( !(fcst_mtddf->grid() == grid) ) { + if ( !(input._grid == grid) ) { mlog << Debug(1) << "Regridding forecast " << engine.conf_info.Fcst->var_info->magic_str() << " to the verification grid.\n"; - Fcst_sd.data = met_regrid(Fcst_sd.data, fcst_mtddf->grid(), grid, + Fcst_sd.data = met_regrid(Fcst_sd.data, input._grid, grid, engine.conf_info.Fcst->var_info->regrid()); } @@ -676,10 +479,6 @@ void ModeExecutive::setup_fcst_data(const Grid &verification_grid) data_min = fmin; data_max = fmax; - // Process percentile thresholds - - engine.conf_info.set_perc_thresh(Fcst_sd.data); - // store the input data units funits = engine.conf_info.Fcst->var_info->units(); ounits = na_str; @@ -698,9 +497,8 @@ void ModeExecutive::setup_fcst_data(const Grid &verification_grid) /////////////////////////////////////////////////////////////////////// - -void ModeExecutive::setup_obs_data(const Grid &verification_grid) - +void ModeExecutive::setup_multivar_obs_data(const Grid &verification_grid, + const ModeInputData &input) { // ShapeData fcst_sd, obs_sd; @@ -709,15 +507,7 @@ void ModeExecutive::setup_obs_data(const Grid &verification_grid) Obs_sd.clear(); // Read the gridded data from the input observation file - - if ( !(obs_mtddf->data_plane(*(engine.conf_info.Obs->var_info), Obs_sd.data)) ) { - - mlog << Error << "\nModeExecutive::setup_obs_data() -> " - << "can't get observation data \"" - << engine.conf_info.Obs->var_info->magic_str() - << "\" from file \"" << obs_file << "\"\n\n"; - exit(1); - } + Obs_sd.data = input._dataPlane; grid = verification_grid; @@ -727,11 +517,11 @@ void ModeExecutive::setup_obs_data(const Grid &verification_grid) // Regrid, if necessary - if ( !(obs_mtddf->grid() == grid) ) { + if ( !(input._grid == grid) ) { mlog << Debug(1) << "Regridding observation " << engine.conf_info.Obs->var_info->magic_str() << " to the verification grid.\n"; - Obs_sd.data = met_regrid(Obs_sd.data, obs_mtddf->grid(), grid, + Obs_sd.data = met_regrid(Obs_sd.data, input._grid, grid, engine.conf_info.Obs->var_info->regrid()); } @@ -755,10 +545,6 @@ void ModeExecutive::setup_obs_data(const Grid &verification_grid) data_min = omin; data_max = omax; - // Process percentile thresholds - - engine.conf_info.set_perc_thresh(Obs_sd.data); - // store the input data units funits = na_str; ounits = engine.conf_info.Obs->var_info->units(); @@ -777,7 +563,7 @@ void ModeExecutive::setup_obs_data(const Grid &verification_grid) /////////////////////////////////////////////////////////////////////// -void ModeExecutive::setup_fcst_obs_data_multivar_intensities(const MultiVarData &mvdf, +void ModeExecutive::setup_multivar_fcst_obs_data_intensities(const MultiVarData &mvdf, const MultiVarData &mvdo) { double fmin, fmax, omin, omax; @@ -789,17 +575,11 @@ void ModeExecutive::setup_fcst_obs_data_multivar_intensities(const MultiVarData Obs_sd.debug_examine(); grid = *(mvdf._grid); - // do not need to read any data, it is stored in the mvd input - // the verification grid was created in the first pass, so we have that as well - // Store the grid engine.set_grid(&grid); - // regridding of inputs is not needed in the second pass, as regridded data outputs - // are stored to input mvd - - // Rescale probabilites from [0, 100] to [0, 1] also not neede in the second pass + // Rescale probabilites from [0, 100] to [0, 1] not needed in the second pass // Check that the valid times match @@ -855,15 +635,12 @@ void ModeExecutive::setup_fcst_obs_data_multivar_intensities(const MultiVarData // in case perc thresh was done, retrieve the values created in the // first pass, which have been stored in mvdo and mvdf + // NOTE: this might be removable with new design engine.conf_info.Obs->conv_thresh_array = mvdo._merge->_convThreshArray; engine.conf_info.Obs->merge_thresh_array = mvdo._merge->_mergeThreshArray; engine.conf_info.Fcst->conv_thresh_array = mvdf._merge->_convThreshArray; engine.conf_info.Fcst->merge_thresh_array = mvdf._merge->_mergeThreshArray; - // This doesn't work with perc_thresh, it uses values restricted to within - // superobjects, which gives different thresholds, so we do the above instead. - //engine.conf_info.set_perc_thresh(Fcst_sd.data, Obs_sd.data); - // masking of inputs not needed, as it was done in the first pass // and stored to Fcst_sd and Obs_sd @@ -887,7 +664,8 @@ void ModeExecutive::setup_fcst_obs_data_multivar_intensities(const MultiVarData /////////////////////////////////////////////////////////////////////// -void ModeExecutive::setup_fcst_obs_data_multivar_super(ShapeData &f_super, ShapeData &o_super, +void ModeExecutive::setup_multivar_fcst_obs_data_super(const ShapeData &f_super, + const ShapeData &o_super, const Grid &igrid) { double fmin, omin, fmax, omax; @@ -972,10 +750,6 @@ void ModeExecutive::setup_fcst_obs_data_multivar_super(ShapeData &f_super, Shape else if(!is_bad_data(fmax) && is_bad_data(omax)) data_max = fmax; else if( is_bad_data(fmax) && !is_bad_data(omax)) data_max = omax; - // Process percentile thresholds...Not applicable for this pass - // do this in the second pass using masked Fcst_sd and Obs_sd - // engine.conf_info.set_perc_thresh(Fcst_sd.data, Obs_sd.data); - // // done // @@ -988,7 +762,7 @@ void ModeExecutive::setup_fcst_obs_data_multivar_super(ShapeData &f_super, Shape /////////////////////////////////////////////////////////////////////// -void ModeExecutive::do_conv_thresh(const int r_index, const int t_index) +void ModeExecutive::do_conv_thresh_traditional(const int r_index, const int t_index) { ModeConfInfo & conf = engine.conf_info; @@ -1000,65 +774,162 @@ void ModeExecutive::do_conv_thresh(const int r_index, const int t_index) R_index = r_index; T_index = t_index; - if (ptype == MULTIVAR_SIMPLE_MERGE) { - conf.set_conv_thresh_by_merge_index(T_index); - } else if (ptype == MULTIVAR_SUPER) { - SingleThresh s("ne-9999"); - conf.set_conv_thresh(s); - conf.set_conv_radius(0.0); + conf.set_conv_radius_by_index(R_index); + conf.set_conv_thresh_by_index(T_index); + conf.set_merge_thresh_by_index(T_index); + + // + // Set up the engine with these raw fields + // + + string what = "forecast and observation fields"; + mlog << Debug(2) << "Identifying objects in the " << what << "...\n"; + + if ( r_index != local_r_index ) { + // need to do convolution + engine.set(Fcst_sd, Obs_sd); } else { - // this could break down if obs and fcst have different arrays - // currently not allowed in multivar mode, should work correctly - // for traditional mode - conf.set_conv_radius_by_index(R_index); - conf.set_conv_thresh_by_index(T_index); + // don't need to do convolution + engine.set_no_conv(Fcst_sd, Obs_sd); } + // + // Compute the contingency table statistics for the fields, if needed + // + if ( conf.ct_stats_flag ) compute_ct_stats(); + + // + // done + // + + local_r_index = r_index; + + return; + +} + +/////////////////////////////////////////////////////////////////////// + +void ModeExecutive::do_conv_thresh_multivar_super() +{ + + ModeConfInfo & conf = engine.conf_info; + + R_index = 0; + T_index = 0; + + SingleThresh s("ne-9999"); + conf.set_conv_thresh(s); + conf.set_conv_radius(0.0); conf.set_merge_thresh_by_index(T_index); // // Set up the engine with these raw fields // - string what = "forecast and observation fields"; - if (ptype == MULTIVAR_SIMPLE || ptype == MULTIVAR_SIMPLE_MERGE) { - if (conf.data_type == ModeDataType_MvMode_Obs) { - what = "observation field"; - } else { - what = "forecast field"; - } - } mlog << Debug(2) << "Identifying objects in the " << what << "...\n"; - if ( r_index != local_r_index ) { // need to do convolution + engine.set_only_split(Fcst_sd, Obs_sd); - if (ptype == MULTIVAR_INTENSITY || ptype == MULTIVAR_SUPER) { - engine.set_only_split(Fcst_sd, Obs_sd); - } else {// MULTIVAR_SIMPLE, MULTIVAR_SIMPLE_MERGE, TRADITIONAL - engine.set(Fcst_sd, Obs_sd); - } - } else { // don't need to do convolution + // + // Compute the contingency table statistics for the fields, if needed + // (not needed for simple or merge, only one field) + // + if ( conf.ct_stats_flag ) compute_ct_stats(); - if (ptype == MULTIVAR_INTENSITY || ptype == MULTIVAR_SUPER) { - engine.set_only_split(Fcst_sd, Obs_sd); - } else {// MULTIVAR_SIMPLE, MULTIVAR_SIMPLE_MERGE, TRADITIONAL - engine.set_no_conv(Fcst_sd, Obs_sd); - } - } + // + // done + // + + local_r_index = 0; + + return; + +} + + +/////////////////////////////////////////////////////////////////////// + + +void ModeExecutive::do_conv_thresh_multivar_intensity_compare() +{ + + ModeConfInfo & conf = engine.conf_info; + + R_index = 0; + T_index = 0; + + conf.set_conv_radius_by_index(R_index); + conf.set_conv_thresh_by_index(T_index); + conf.set_merge_thresh_by_index(T_index); + + // + // Set up the engine with these raw fields + // + + string what = "forecast and observation fields"; + mlog << Debug(2) << "Identifying objects in the " << what << "...\n"; + + engine.set_only_split(Fcst_sd, Obs_sd); // // Compute the contingency table statistics for the fields, if needed - // (not needed for simple or merge, only one field) // - if (ptype != MULTIVAR_SIMPLE && ptype != MULTIVAR_SIMPLE_MERGE) { - if ( conf.ct_stats_flag ) compute_ct_stats(); - } + if ( conf.ct_stats_flag ) compute_ct_stats(); // // done // - local_r_index = r_index; + local_r_index = 0; + + return; + +} + +/////////////////////////////////////////////////////////////////////// + +void ModeExecutive::do_conv_thresh_multivar_simple(Processing_t p) +{ + + ModeConfInfo & conf = engine.conf_info; + + R_index = 0; + T_index = 0; + + if (p == MULTIVAR_SIMPLE_MERGE) { + conf.set_conv_thresh_by_merge_index(T_index); + } else if (p == MULTIVAR_SIMPLE) { + conf.set_conv_radius_by_index(0); + conf.set_conv_thresh_by_index(0); + } else { + mlog << Error << "\nModeExecutive::do_conv_thresh_multivar_simple() -> " + << "Wrong processing type input " << stype(p) << "\"\n\n"; + exit(1); + } + + conf.set_merge_thresh_by_index(0); + + // + // Set up the engine with these raw fields + // + + string what; + if (conf.data_type == ModeDataType_MvMode_Obs) { + what = "observation field"; + } else { + what = "forecast field"; + } + mlog << Debug(2) << "Identifying objects in the " << what << "...\n"; + + engine.set(Fcst_sd, Obs_sd); + + // (ct_stats not needed for simple or merge, only one field) + // + // done + // + + local_r_index = 0; return; @@ -1073,7 +944,7 @@ void ModeExecutive::clear_internal_r_index() /////////////////////////////////////////////////////////////////////// -void ModeExecutive::do_merging() +void ModeExecutive::do_merging_traditional() { mlog << Debug(2) << "Identified: " << engine.n_fcst << " forecast objects " @@ -1105,10 +976,11 @@ void ModeExecutive::do_merging() /////////////////////////////////////////////////////////////////////// -void ModeExecutive::do_merging(ShapeData &f_merge, ShapeData &o_merge) - +void ModeExecutive::do_merging_multivar(const ShapeData &f_merge, + const ShapeData &o_merge, + Processing_t p) { - if (ptype == MULTIVAR_SUPER) { + if (p == MULTIVAR_SUPER) { // set the merge flag and merge_thresh appropriately ModeConfInfo & conf = engine.conf_info; SingleThresh s("ne-9999"); @@ -1116,14 +988,11 @@ void ModeExecutive::do_merging(ShapeData &f_merge, ShapeData &o_merge) conf.set_fcst_merge_thresh(s); conf.set_obs_merge_flag(MergeType_Thresh); conf.set_obs_merge_thresh(s); - } else { - if (ptype != MULTIVAR_INTENSITY) { - mlog << Error << "\nModeExecutive::do_merging(shapedata, shapedata) -> " - << "wrong method for processing type " << stype(ptype) << "\n\n"; - exit(1); - } + } else if (p != MULTIVAR_INTENSITY) { + mlog << Error << "\nModeExecutive::do_merging(shapedata, shapedata, p) -> " + << "wrong method for processing type " << stype(p) << "\n\n"; + exit(1); } - mlog << Debug(2) << "Identified: " << engine.n_fcst << " forecast objects " @@ -1136,8 +1005,7 @@ void ModeExecutive::do_merging(ShapeData &f_merge, ShapeData &o_merge) // Do the forecast merging - engine.do_fcst_merging(default_config_file.c_str(), merge_config_file.c_str(), - f_merge); + engine.do_fcst_merging(f_merge); mlog << Debug(2) << "Performing merging (" @@ -1146,8 +1014,7 @@ void ModeExecutive::do_merging(ShapeData &f_merge, ShapeData &o_merge) // Do the observation merging - engine.do_obs_merging(default_config_file.c_str(), merge_config_file.c_str(), - o_merge); + engine.do_obs_merging(o_merge); mlog << Debug(2) << "Remaining: " << engine.n_fcst << " forecast objects " @@ -1158,10 +1025,12 @@ void ModeExecutive::do_merging(ShapeData &f_merge, ShapeData &o_merge) /////////////////////////////////////////////////////////////////////// -void ModeExecutive::do_match_merge(ShapeData &f_merge, ShapeData &o_merge) +void ModeExecutive::do_match_merge_multivar(const ShapeData &f_merge, + const ShapeData &o_merge, + Processing_t p) { - do_merging(f_merge, o_merge); + do_merging_multivar(f_merge, o_merge, p); // Do the matching of objects between fields @@ -1174,10 +1043,10 @@ void ModeExecutive::do_match_merge(ShapeData &f_merge, ShapeData &o_merge) /////////////////////////////////////////////////////////////////////// -void ModeExecutive::do_match_merge() +void ModeExecutive::do_match_merge_traditional() { - do_merging(); + do_merging_traditional(); // Do the matching of objects between fields @@ -1348,82 +1217,104 @@ void ModeExecutive::set_raw_to_full(float *fcst_raw_data, /////////////////////////////////////////////////////////////////////// -void ModeExecutive::process_output(const MultiVarData *mvdf, - const MultiVarData *mvdo) - +void ModeExecutive::process_output_multivar_super() { + isMultivarOutput = false; + isMultivarSuperOutput = true; - // store to class member so don't have to pass it around - isMultivarOutput = (ptype == MULTIVAR_INTENSITY); - isMultivarSuperOutput = (ptype == MULTIVAR_SUPER); + // use the configured multivar name and level + fcst_magic_string = engine.conf_info.fcst_multivar_name.string() + "_" + engine.conf_info.fcst_multivar_level.string(); + obs_magic_string = engine.conf_info.obs_multivar_name.string() + "_" + engine.conf_info.obs_multivar_level.string(); - if (isMultivarOutput) { + // Create output stats files and plots - // get the magic strings, which will be used in file naming - fcst_magic_string = engine.conf_info.Fcst->var_info->magic_str().c_str(); - obs_magic_string = engine.conf_info.Obs->var_info->magic_str().c_str(); + write_obj_stats(); - // replace forward slashes with underscores to prevent new directories - replace(fcst_magic_string.begin(), fcst_magic_string.end(), '/', '_'); - replace(obs_magic_string.begin(), obs_magic_string.end(), '/', '_'); + if ( engine.conf_info.ct_stats_flag ) write_ct_stats(); + + write_obj_netcdf(engine.conf_info.nc_info); - // replace (*,*) with '_all_all_' - replaceAll(fcst_magic_string, "*", "all"); - replaceAll(obs_magic_string, "*", "all"); - replaceAll(fcst_magic_string, ",", "_"); - replaceAll(obs_magic_string, ",", "_"); - replaceAll(fcst_magic_string, "(", "_"); - replaceAll(obs_magic_string, "(", "_"); - replaceAll(fcst_magic_string, ")", ""); - replaceAll(obs_magic_string, ")", ""); - - } else if (isMultivarSuperOutput) { + if ( engine.conf_info.ps_plot_flag ) plot_engine(); - // use the configured multivar name and level - fcst_magic_string = engine.conf_info.fcst_multivar_name.string() + "_" + engine.conf_info.fcst_multivar_level.string(); - obs_magic_string = engine.conf_info.obs_multivar_name.string() + "_" + engine.conf_info.obs_multivar_level.string(); + return; - } else { +} - // just in case make these empty - fcst_magic_string = ""; - obs_magic_string = ""; +/////////////////////////////////////////////////////////////////////// - } - +void ModeExecutive::process_output_multivar_intensity_compare(const MultiVarData *mvdf, + const MultiVarData *mvdo) + +{ + isMultivarOutput = true; + isMultivarSuperOutput = false; + + // get the magic strings, which will be used in file naming + fcst_magic_string = engine.conf_info.Fcst->var_info->magic_str().c_str(); + obs_magic_string = engine.conf_info.Obs->var_info->magic_str().c_str(); + + // replace forward slashes with underscores to prevent new directories + replace(fcst_magic_string.begin(), fcst_magic_string.end(), '/', '_'); + replace(obs_magic_string.begin(), obs_magic_string.end(), '/', '_'); + + // replace (*,*) with '_all_all_' + replaceAll(fcst_magic_string, "*", "all"); + replaceAll(obs_magic_string, "*", "all"); + replaceAll(fcst_magic_string, ",", "_"); + replaceAll(obs_magic_string, ",", "_"); + replaceAll(fcst_magic_string, "(", "_"); + replaceAll(obs_magic_string, "(", "_"); + replaceAll(fcst_magic_string, ")", ""); + replaceAll(obs_magic_string, ")", ""); + // Create output stats files and plots write_obj_stats(); if ( engine.conf_info.ct_stats_flag ) write_ct_stats(); - - if (isMultivarOutput) { - if (mvdf && mvdo) { - double fmin = mvdf->_data_min; - double fmax = mvdf->_data_max; - double omin = mvdo->_data_min; - double omax = mvdo->_data_max; - double data_min, data_max; - if (!is_bad_data(fmin) && !is_bad_data(omin)) data_min = min(fmin, omin); - else if(!is_bad_data(fmin) && is_bad_data(omin)) data_min = fmin; - else if( is_bad_data(fmin) && !is_bad_data(omin)) data_min = omin; - - if (!is_bad_data(fmax) && !is_bad_data(omax)) data_max = max(fmax, omax); - else if(!is_bad_data(fmax) && is_bad_data(omax)) data_max = fmax; - else if( is_bad_data(fmax) && !is_bad_data(omax)) data_max = omax; + double fmin = mvdf->_data_min; + double fmax = mvdf->_data_max; + double omin = mvdo->_data_min; + double omax = mvdo->_data_max; + double data_min, data_max; + if (!is_bad_data(fmin) && !is_bad_data(omin)) data_min = min(fmin, omin); + else if(!is_bad_data(fmin) && is_bad_data(omin)) data_min = fmin; + else if( is_bad_data(fmin) && !is_bad_data(omin)) data_min = omin; + + if (!is_bad_data(fmax) && !is_bad_data(omax)) data_max = max(fmax, omax); + else if(!is_bad_data(fmax) && is_bad_data(omax)) data_max = fmax; + else if( is_bad_data(fmax) && !is_bad_data(omax)) data_max = omax; - set_raw_to_full(mvdf->_simple->_raw_data, - mvdo->_simple->_raw_data, - mvdf->_nx, mvdf->_ny, - data_min, data_max); - } else { - mlog << Error << "\nModeExecutive::process_output() -> " - << "no multivar data when multivar data is expected\n\n"; - exit(1); - } - } + set_raw_to_full(mvdf->_simple->_raw_data,mvdo->_simple->_raw_data, + mvdf->_nx, mvdf->_ny, data_min, data_max); + + write_obj_netcdf(engine.conf_info.nc_info); + + if ( engine.conf_info.ps_plot_flag ) plot_engine(); + + return; + +} + +/////////////////////////////////////////////////////////////////////// + +void ModeExecutive::process_output_traditional() +{ + isMultivarOutput = false; + isMultivarSuperOutput = false; + + // just in case make these empty + fcst_magic_string = ""; + obs_magic_string = ""; + + // Create output stats files and plots + + write_obj_stats(); + + if ( engine.conf_info.ct_stats_flag ) write_ct_stats(); + write_obj_netcdf(engine.conf_info.nc_info); if ( engine.conf_info.ps_plot_flag ) plot_engine(); @@ -1737,7 +1628,8 @@ MultiVarData *ModeExecutive::get_multivar_data(ModeDataType dtype) obs_magic_string = engine.conf_info.Obs->var_info->magic_str().c_str(); // replace forward slashes with underscores to prevent new directories replace(obs_magic_string.begin(), obs_magic_string.end(), '/', '_'); - mvd->init(dtype, obs_magic_string, grid, otype, ounits, olevel, data_min, data_max); + mvd->init(dtype, obs_magic_string, grid, + ounits, olevel, data_min, data_max); mvd->set_obj(engine.obs_split, simple); mvd->set_raw(engine.obs_raw, simple); mvd->set_shapedata(Obs_sd, simple); @@ -1748,7 +1640,8 @@ MultiVarData *ModeExecutive::get_multivar_data(ModeDataType dtype) fcst_magic_string = engine.conf_info.Fcst->var_info->magic_str().c_str(); // replace forward slashes with underscores to prevent new directories replace(fcst_magic_string.begin(), fcst_magic_string.end(), '/', '_'); - mvd->init(dtype, fcst_magic_string, grid, ftype, funits, flevel, data_min, data_max); + mvd->init(dtype, fcst_magic_string, grid, + funits, flevel, data_min, data_max); mvd->set_obj(engine.fcst_split, simple); mvd->set_raw(engine.fcst_raw, simple); mvd->set_shapedata(Fcst_sd, simple); @@ -2738,7 +2631,7 @@ void ModeExecutive::write_ct_stats() /////////////////////////////////////////////////////////////////////// -void ModeExecutive::conf_read(const string &default_config_filename) +void ModeExecutive::conf_read() { // Create the default config file name diff --git a/src/tools/core/mode/mode_exec.h b/src/tools/core/mode/mode_exec.h index 34c8d7fb5e..f3c9937d35 100644 --- a/src/tools/core/mode/mode_exec.h +++ b/src/tools/core/mode/mode_exec.h @@ -75,83 +75,118 @@ class ModeExecutive { // the various mode algorithm settings typedef enum {TRADITIONAL, MULTIVAR_SIMPLE, MULTIVAR_SIMPLE_MERGE, MULTIVAR_INTENSITY, MULTIVAR_SUPER} Processing_t; - ModeExecutive(Processing_t p=TRADITIONAL); + ModeExecutive(); ~ModeExecutive(); void clear(); void init_traditional(int n_files); - void init_multivar_verif_grid(); - void init_multivar_simple(int n_files, ModeDataType dtype); - void init_multivar_intensities(GrdFileType ftype, GrdFileType otype); - void check_multivar_perc_thresh_settings(); - + void init_multivar_simple(int j, int n_files, ModeDataType dtype, const ModeConfInfo &conf); + void init_multivar_intensities(const ModeConfInfo &conf); int n_conv_radii () const; int n_conv_threshs () const; - int n_runs() const; + // these are used only for traditional mode int R_index; // indices into the convolution radius and threshold arrays int T_index; // for the current run // - // Input configuration files + // Input configuration files, all 3 used only for traditional mode + // Multivar mode handles the configs outside of the exec // - + // the hardwired default traditional mode config ConcatString default_config_file; - ConcatString match_config_file; + + // set for both trad and multivar, this is the default file on the command line + // used only for traditional + ConcatString match_config_file; + + // the extra one that can be set only for traditional mode ConcatString merge_config_file; // - // Input files + // Input filenames, set for both multivar and trad mode + // but used only for trad mode // - ConcatString fcst_file; ConcatString obs_file; + + // set and used only for trad mode Met2dDataFile * fcst_mtddf; Met2dDataFile * obs_mtddf; + // used for both trad and multivar mode TTContingencyTable cts[n_cts]; + // used for both trad and multivar mode ModeFuzzyEngine engine; // verification grid + // used for both trad and multivar mode, set by both Grid grid; Box xy_bb; ConcatString out_dir; + + // set for both trad and multivar mode, used for plotting limits double data_min, data_max; + // set for trad and multivar, used in the engine mode algorithm ShapeData Fcst_sd, Obs_sd; + // not used by multivar GrdFileType ftype, otype; + + // set into execs's conf varInfo object, only for multivar intensity comparisons + // for trad it's read in from the config string funits, ounits; + + // set into execs's conf varInfo object, only for multivar intensity comparisons + // for trad it's read in from the config string flevel, olevel; + // used in multivar only to customize outputs correctly bool isMultivarOutput; bool isMultivarSuperOutput; - Processing_t ptype; - + void setup_verification_grid(const ModeInputData &fcst, + const ModeInputData &obs, + const ModeConfInfo &conf); + void clear_internal_r_index(); - void setup_verification_grid(); - void setup_fcst_obs_data_traditional(); - void setup_fcst_data(const Grid &verification_grid); - void setup_obs_data(const Grid &verification_grid); - void setup_fcst_obs_data_multivar_intensities(const MultiVarData &mvdf, const MultiVarData &mvdo); - void setup_fcst_obs_data_multivar_super(ShapeData &f_super, ShapeData &o_super, const Grid &igrid); - void do_conv_thresh(const int r_index, const int t_index); - void do_merging(); - void do_merging(ShapeData &f_merge, ShapeData &o_merge); - void do_match_merge(); - void do_match_merge(ShapeData &f_merge, ShapeData &o_merge); + + void setup_traditional_fcst_obs_data(); + void setup_multivar_fcst_data(const Grid &verification_grid, const ModeInputData &input); + void setup_multivar_obs_data(const Grid &verification_grid, const ModeInputData &input); + void setup_multivar_fcst_obs_data_intensities(const MultiVarData &mvdf, + const MultiVarData &mvdo); + void setup_multivar_fcst_obs_data_super(const ShapeData &f_super, + const ShapeData &o_super, + const Grid &igrid); + + void do_conv_thresh_traditional(const int r_index, const int t_index); + void do_conv_thresh_multivar_super(); + void do_conv_thresh_multivar_intensity_compare(); + void do_conv_thresh_multivar_simple(Processing_t p); + + void do_merging_traditional(); + void do_merging_multivar(const ShapeData &f_merge, const ShapeData &o_merge, + Processing_t p); + + void do_match_merge_traditional(); + void do_match_merge_multivar(const ShapeData &f_merge, const ShapeData &o_merge, + Processing_t p); void process_masks(ShapeData &, ShapeData &); void process_fcst_masks(ShapeData &); void process_obs_masks(ShapeData &); - void process_output(const MultiVarData *mvdf=NULL, - const MultiVarData *mvdo=NULL); + + void process_output_traditional(); + void process_output_multivar_intensity_compare(const MultiVarData *mvdf, + const MultiVarData *mvdo); + void process_output_multivar_super(); void set_raw_to_full(float *fcst_raw_data, float *obs_raw_data, @@ -178,7 +213,9 @@ class ModeExecutive { void write_poly_netcdf(netCDF::NcFile *, const ObjPolyType); void write_ct_stats(); - void conf_read(const string &default_config_filename); + // traditional only, multivar reads outside of the exec + void conf_read(); + static string stype(Processing_t t); }; diff --git a/src/tools/core/mode/mode_frontend.cc b/src/tools/core/mode/mode_frontend.cc index 456ae71f7a..4831cb59d2 100644 --- a/src/tools/core/mode/mode_frontend.cc +++ b/src/tools/core/mode/mode_frontend.cc @@ -35,9 +35,10 @@ using namespace std; extern const char * const program_name; static ModeExecutive *mode_exec = 0; -static ModeExecutive::Processing_t ptype = ModeExecutive::TRADITIONAL; -static int compress_level = -1; +// used only for traditional mode, multivar sets it into config previous +// to the frontend creation +static int compress_level = -1; /////////////////////////////////////////////////////////////////////// @@ -61,160 +62,13 @@ ModeFrontEnd::~ModeFrontEnd() } -/////////////////////////////////////////////////////////////////////// - - -Grid ModeFrontEnd::create_verification_grid(const StringArray & Argv) - -{ - if ( mode_exec ) { delete mode_exec; mode_exec = 0; } - mode_exec = new ModeExecutive; - compress_level = -1; - - // - // Process the command line arguments - // - - process_command_line(Argv, false); - - mode_exec->init_multivar_verif_grid(); - - ModeConfInfo & conf = mode_exec->engine.conf_info; - conf.set_field_index(0); - if (compress_level >= 0) conf.nc_info.set_compress_level(compress_level); - - - // - // read in data (Note multiple reads of same data) - // - mode_exec->setup_verification_grid(); - Grid g = mode_exec->grid; - delete mode_exec; mode_exec = 0; - return g; -} - - -/////////////////////////////////////////////////////////////////////// - -int ModeFrontEnd::create_multivar_simple_objects(const StringArray & Argv, ModeDataType dtype, - const Grid &verification_grid, int field_index, int n_files) - -{ - init(ModeExecutive::MULTIVAR_SIMPLE); - - // - // Process the command line arguments - // - - process_command_line_for_simple_objects(Argv, dtype); - - mode_exec->init_multivar_simple(n_files, dtype); - - ModeConfInfo & conf = mode_exec->engine.conf_info; - if ( field_index >= 0 ) conf.set_field_index(field_index); - if (compress_level >= 0) conf.nc_info.set_compress_level(compress_level); - - // need to do this after setting field index above - mode_exec->check_multivar_perc_thresh_settings(); - - // - // read in data (Note multiple reads of data) - // - - if (dtype == ModeDataType_MvMode_Fcst) { - mode_exec->setup_fcst_data(verification_grid); - } else { - mode_exec->setup_obs_data(verification_grid); - } - - // - // mode algorithm - // - if ( conf.quilt ) { - - do_quilt(); - - } else { - - do_straight(); - - } - - // - // done - // - -#ifdef WITH_PYTHON - GP.finalize(); -#endif - return (0); -} - -/////////////////////////////////////////////////////////////////////// - -int ModeFrontEnd::create_multivar_merge_objects(const StringArray & Argv, ModeDataType dtype, - const Grid &verification_grid, int field_index, - int n_files) - -{ - init(ModeExecutive::MULTIVAR_SIMPLE_MERGE); - - // - // Process the command line arguments - // - - process_command_line_for_simple_objects(Argv, dtype); - - mode_exec->init_multivar_simple(n_files, dtype); - - ModeConfInfo & conf = mode_exec->engine.conf_info; - if ( field_index >= 0 ) conf.set_field_index(field_index); - if (compress_level >= 0) conf.nc_info.set_compress_level(compress_level); - - // need to do this after setting field index above - mode_exec->check_multivar_perc_thresh_settings(); - - - // - // read in data (Note multiple reads not desired) - // - - if (dtype == ModeDataType_MvMode_Fcst) { - mode_exec->setup_fcst_data(verification_grid); - } else { - mode_exec->setup_obs_data(verification_grid); - } - - - // - // mode algorithm - // - if ( conf.quilt ) { - - do_quilt(); - - } else { - - do_straight(); - - } - - // - // done - // - -#ifdef WITH_PYTHON - GP.finalize(); -#endif - return (0); -} /////////////////////////////////////////////////////////////////////// int ModeFrontEnd::run_traditional(const StringArray & Argv) { - init(ModeExecutive::TRADITIONAL); + init(); int field_index = -1; int n_files = 1; @@ -223,7 +77,7 @@ int ModeFrontEnd::run_traditional(const StringArray & Argv) // Process the command line arguments // - process_command_line(Argv, false); + process_command_line(Argv); mode_exec->init_traditional(n_files); @@ -236,7 +90,7 @@ int ModeFrontEnd::run_traditional(const StringArray & Argv) // read in data // - mode_exec->setup_fcst_obs_data_traditional(); + mode_exec->setup_traditional_fcst_obs_data(); // // mode algorithm @@ -263,129 +117,6 @@ int ModeFrontEnd::run_traditional(const StringArray & Argv) /////////////////////////////////////////////////////////////////////// -int ModeFrontEnd::multivar_intensity_comparisons(const StringArray & Argv, const MultiVarData &mvdf, - const MultiVarData &mvdo, bool has_union_f, - bool has_union_o, ShapeData &merge_f, - ShapeData &merge_o, int field_index_f, int field_index_o) -{ - init(ModeExecutive::MULTIVAR_INTENSITY); - - // - // Process the command line arguments - // - - process_command_line(Argv, false); - - mode_exec->init_multivar_intensities(mvdf._type, mvdo._type); - - ModeConfInfo & conf = mode_exec->engine.conf_info; - if (compress_level >= 0) conf.nc_info.set_compress_level(compress_level); - conf.set_field_index(field_index_f, field_index_o); - - // for multivar intensities, explicity set the level and units using stored values - // from pass1 - conf.Fcst->var_info->set_level_name(mvdf._level.c_str()); - conf.Fcst->var_info->set_units(mvdf._units.c_str()); - if (has_union_f && conf.Fcst->merge_flag == MergeType_Thresh) { - mlog << Warning << "\nModeFrontEnd::multivar_intensity_comparisons() -> " - << "Logic includes union '||' along with 'merge_flag=THRESH' " - << ". This can lead to bad results\n\n"; - } - conf.Obs->var_info->set_level_name(mvdo._level.c_str()); - conf.Obs->var_info->set_units(mvdo._units.c_str()); - if (has_union_o && conf.Obs->merge_flag == MergeType_Thresh) { - mlog << Warning << "\nModeFrontEnd::multivar_intensity_comparisons() -> " - << "Logic includes union '||' along with 'merge_flag=THRESH' " - << ". This can lead to bad results\n\n"; - } - - // - // set up data access using inputs - // - mode_exec->setup_fcst_obs_data_multivar_intensities(mvdf, mvdo); - - // - // run the mode algorithm - // - - if ( conf.quilt ) { - - do_quilt(); - - } else { - - do_straight_multivar_intensity(mvdf, mvdo, merge_f, merge_o); - - } - - // - // done - // - -#ifdef WITH_PYTHON - GP.finalize(); -#endif - return (0); -} - -/////////////////////////////////////////////////////////////////////// - -int ModeFrontEnd::run_super(const StringArray & Argv, - ShapeData &f_super, ShapeData &o_super, - ShapeData &f_merge, ShapeData &o_merge, - GrdFileType ftype, GrdFileType otype, const Grid &grid, - bool has_union) -{ - init(ModeExecutive::MULTIVAR_SUPER); - - // - // Process the command line arguments - // - - process_command_line(Argv, true); - - mode_exec->init_multivar_intensities(ftype, otype); - - ModeConfInfo & conf = mode_exec->engine.conf_info; - if (compress_level >= 0) conf.nc_info.set_compress_level(compress_level); - if (has_union && (conf.Fcst->merge_flag == MergeType_Thresh || - conf.Obs->merge_flag == MergeType_Thresh)) { - mlog << Warning << "\nModeFrontEnd::run_super() -> " - << "Logic includes union '||' along with 'merge_flag=THRESH' " - << ". This can lead to bad results\n\n"; - } - - // - // set up data access using inputs - // - mode_exec->setup_fcst_obs_data_multivar_super(f_super, o_super, grid); - - // - // run the mode algorithm - // - - if ( conf.quilt ) { - - do_quilt(); - - } else { - - do_straight_multivar_super(f_merge, o_merge); - - } - - // - // done - // - -#ifdef WITH_PYTHON - GP.finalize(); -#endif - return (0); -} - -/////////////////////////////////////////////////////////////////////// - void ModeFrontEnd::do_straight() @@ -398,78 +129,9 @@ void ModeFrontEnd::do_straight() for (int index=0; indexdo_conv_thresh(index, index); - if (ptype == ModeExecutive::TRADITIONAL) { - - mode_exec->do_match_merge(); - mode_exec->process_output(); - } - } - - mode_exec->clear_internal_r_index(); - - // - // done - // - - return; - -} - -/////////////////////////////////////////////////////////////////////// - - -void ModeFrontEnd::do_straight_multivar_intensity(const MultiVarData &mvdf, - const MultiVarData &mvdo, - ShapeData &f_merge, - ShapeData &o_merge) - -{ - int NCT, NCR; - - do_straight_init(NCT, NCR); - - mode_exec->clear_internal_r_index(); - - for (int index=0; indexdo_conv_thresh(index, index); - mode_exec->do_match_merge(f_merge, o_merge); - - // here replace raw data and min/max for plotting - - mode_exec->process_output(&mvdf, &mvdo); - } - - mode_exec->clear_internal_r_index(); - - // - // done - // - - return; - -} - - - -/////////////////////////////////////////////////////////////////////// - - -void ModeFrontEnd::do_straight_multivar_super(ShapeData &f_merge, ShapeData &o_merge) - -{ - int NCT, NCR; - - do_straight_init(NCT, NCR); - - mode_exec->clear_internal_r_index(); - - for (int index=0; indexdo_conv_thresh(index, index); - mode_exec->do_match_merge(f_merge, o_merge); - mode_exec->process_output(); + mode_exec->do_conv_thresh_traditional(index, index); + mode_exec->do_match_merge_traditional(); + mode_exec->process_output_traditional(); } mode_exec->clear_internal_r_index(); @@ -482,19 +144,12 @@ void ModeFrontEnd::do_straight_multivar_super(ShapeData &f_merge, ShapeData &o_m } - /////////////////////////////////////////////////////////////////////// void ModeFrontEnd::do_quilt() { - if (ptype != ModeExecutive::TRADITIONAL) { - mlog << Error << "\nModeFrontend::do_quilt() -> quilting not yet implemented for multivar mode \n\n"; - exit ( 1 ); - } - - int t_index, r_index; // indices into the convolution threshold and radius arrays @@ -503,14 +158,9 @@ void ModeFrontEnd::do_quilt() for (r_index=0; r_index<(mode_exec->n_conv_radii()); ++r_index) { for (t_index=0; t_index<(mode_exec->n_conv_threshs()); ++t_index) { - - mode_exec->do_conv_thresh(r_index, t_index); - - mode_exec->do_match_merge(); - - if (ptype == ModeExecutive::TRADITIONAL) { - mode_exec->process_output(); - } + mode_exec->do_conv_thresh_traditional(r_index, t_index); + mode_exec->do_match_merge_traditional(); + mode_exec->process_output_traditional(); } } @@ -526,29 +176,13 @@ void ModeFrontEnd::do_quilt() /////////////////////////////////////////////////////////////////////// -MultiVarData *ModeFrontEnd::get_multivar_data(ModeDataType dtype) -{ - return mode_exec->get_multivar_data(dtype); -} - - -/////////////////////////////////////////////////////////////////////// - -void ModeFrontEnd::add_multivar_merge_data(MultiVarData *mvdi, ModeDataType dtype) -{ - return mode_exec->add_multivar_merge_data(mvdi, dtype); -} - -/////////////////////////////////////////////////////////////////////// - -void ModeFrontEnd::init(ModeExecutive::Processing_t p) +void ModeFrontEnd::init() { - ptype = p; - mlog << Debug(1) << "Running multivar front end for " << ModeExecutive::stype(ptype) << "\n"; + mlog << Debug(1) << "Running traditional mode front end\n"; if ( mode_exec ) { delete mode_exec; mode_exec = 0; } - mode_exec = new ModeExecutive(ptype); + mode_exec = new ModeExecutive();//ModeExecutive::TRADITIONAL); compress_level = -1; } @@ -569,20 +203,12 @@ void ModeFrontEnd::do_straight_init(int &NCT, int &NCR) const exit ( 1 ); } - - if (NCT > 1 && ptype != ModeExecutive::TRADITIONAL) { - - mlog << Error << "\nModeFrontEnd::do_straight_init() ->" - << ": multiple convolution radii and thresholds not implemented in multivar mode\n\n"; - - exit ( 1 ); - } } /////////////////////////////////////////////////////////////////////// -void ModeFrontEnd::process_command_line(const StringArray & argv, bool ismultivar) +void ModeFrontEnd::process_command_line(const StringArray & argv) { CommandLine cline; ConcatString s; @@ -595,7 +221,7 @@ void ModeFrontEnd::process_command_line(const StringArray & argv, bool ismultiva mode_exec->out_dir = replace_path(default_out_dir); // - // Check for zero arguments (note not correct for multivar mode, want to show multivar_usage + // Check for zero arguments // if(argc == 1) singlevar_usage(); @@ -607,7 +233,7 @@ void ModeFrontEnd::process_command_line(const StringArray & argv, bool ismultiva cline.set(argv); // - // Set the usage function NOTE wrong for multivar, want multivar_usage + // Set the usage function // cline.set_usage(singlevar_usage); @@ -628,103 +254,19 @@ void ModeFrontEnd::process_command_line(const StringArray & argv, bool ismultiva cline.parse(); - if (ismultivar) { - // - // Check for error. There should be 1 argument left: - // config filename - // - if(cline.n() != 1) singlevar_usage(); // wrong need multivar usage - - // - // Store the input forecast and observation file names, placeholders - // - mode_exec->fcst_file = "not set"; - mode_exec->obs_file = "not set"; - mode_exec->match_config_file = cline[0]; - - } else { - // - // Check for error. There should be three arguments left: - // forecast, observation, and config filenames - // - if(cline.n() != 3) singlevar_usage(); - - // - // Store the input forecast and observation file names - // - mode_exec->fcst_file = cline[0]; - mode_exec->obs_file = cline[1]; - mode_exec->match_config_file = cline[2]; - - } -} - - -/////////////////////////////////////////////////////////////////////// - -void ModeFrontEnd::process_command_line_for_simple_objects(const StringArray &argv, ModeDataType dtype) -{ - CommandLine cline; - ConcatString s; - const int argc = argv.n(); - - // - // Set the default output directory - // - - mode_exec->out_dir = replace_path(default_out_dir); - - // - // Check for zero arguments (note not correct for multivar mode, want to show multivar_usage - // - - if(argc == 1) singlevar_usage(); - - // - // Parse the command line into tokens - // - - cline.set(argv); - - // - // Set the usage function NOTE wrong for multivar, want multivar_usage - // - - cline.set_usage(singlevar_usage); - - // - // Add the options function calls - // - - cline.add(set_config_merge_file, "-config_merge", 1); - cline.add(set_outdir, "-outdir", 1); - cline.add(set_logfile, "-log", 1); - cline.add(set_verbosity, "-v", 1); - cline.add(set_compress, "-compress", 1); - // - // Parse the command line + // Check for error. There should be three arguments left: + // forecast, observation, and config filenames // - - cline.parse(); + if(cline.n() != 3) singlevar_usage(); // - // Check for error. There should be two arguments left: - // data and config filenames + // Store the input forecast and observation file names // - if(cline.n() != 2) singlevar_usage(); + mode_exec->fcst_file = cline[0]; + mode_exec->obs_file = cline[1]; + mode_exec->match_config_file = cline[2]; - // - // Store the file name - // - if (dtype == ModeDataType_MvMode_Fcst) { - mode_exec->fcst_file = cline[0]; - mode_exec->obs_file = "None"; - } else { - mode_exec->obs_file = cline[0]; - mode_exec->fcst_file = "None"; - } - mode_exec->match_config_file = cline[1]; } /////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/mode/mode_frontend.h b/src/tools/core/mode/mode_frontend.h index 55ecdf7f50..658f90ff4c 100644 --- a/src/tools/core/mode/mode_frontend.h +++ b/src/tools/core/mode/mode_frontend.h @@ -19,8 +19,6 @@ #include #include "mode_exec.h" #include "string_array.h" -#include "multivar_data.h" -#include "mode_data_type.h" class ModeFrontEnd { @@ -34,54 +32,21 @@ class ModeFrontEnd { string default_out_dir; - Grid create_verification_grid(const StringArray & Argv); - - // run the multivar simple object, where there is only one input data, either forecast or obs - int create_multivar_simple_objects(const StringArray & Argv, ModeDataType dtype, const Grid &verification_grid, - int field_index=-1, int n_files=1); - - // run the multivar simple object merge algorithm, with one input data, either forecast or obs - int create_multivar_merge_objects(const StringArray & Argv, ModeDataType dtype, const Grid &verification_grid, - int field_index=-1, int n_files=1); // run the default single var mode interface (traditional mode) int run_traditional(const StringArray & Argv); - // run the multivar intensity algorithm, where one forecast and one obs are restricted to be within superobjects - // and the traditional mode algorithm compares them - int multivar_intensity_comparisons(const StringArray & Argv, const MultiVarData &mvdf, const MultiVarData &mvdo, - bool has_union_f, bool has_union_o, ShapeData &merge_f, - ShapeData &merge_o, int field_index_f, int field_index_o); - - // multivar superobject interface, with no intensities - int run_super(const StringArray & Argv, ShapeData &f_super, ShapeData &o_super, - ShapeData &f_merge, ShapeData &o_merge, - GrdFileType ftype, GrdFileType otype, const Grid &grid, bool has_union); - + void init(); + // so far only implemented for traditional mode void do_quilt (); - // MODE algorithm for traditional, multivar simple, or multivar merge cases + // MODE algorithm for traditional mode void do_straight (); - // MODE algorithm when doing multivar intensities - void do_straight_multivar_intensity (const MultiVarData &mvdf, - const MultiVarData &mvdo, ShapeData &mergef, - ShapeData &mergeo); - - // MODE algorithm when doing multivar super with no intensities - void do_straight_multivar_super (ShapeData &f_merge, ShapeData &o_merge); - - - MultiVarData *get_multivar_data(ModeDataType dtype); - - void add_multivar_merge_data(MultiVarData *mvdi, ModeDataType dtype); - - void init(ModeExecutive::Processing_t p); void do_straight_init(int &NCT, int &NCR) const; - void process_command_line_for_simple_objects(const StringArray &, ModeDataType dtype); - void process_command_line(const StringArray &, bool is_multivar); + void process_command_line(const StringArray &); static void set_config_merge_file (const StringArray &); static void set_outdir (const StringArray &); diff --git a/src/tools/core/mode/mode_superobject.cc b/src/tools/core/mode/mode_superobject.cc new file mode 100644 index 0000000000..27d1c3092f --- /dev/null +++ b/src/tools/core/mode/mode_superobject.cc @@ -0,0 +1,184 @@ +using namespace std; +#include "mode_superobject.h" +#include "multivar_data.h" + +//////////////////////////////////////////////////////////////////////// +static void _mask_super(const string &name, int nx, int ny, DataPlane &data) +{ + + if (nx != data.nx() || ny != data.ny()) { + mlog << Error << "\nModeSuperObject::mask_data_super() -> " << name + << " :dimensions don't match " << nx << " " << ny + << " " << data.nx() << " " << data.ny() << "\n\n"; + + exit( 1 ); + } + + int nmasked=0, nkeep=0; + + for (int x=0; x " << name + << " :dimensions don't match " << nx << " " << ny + << " " << data.nx() << " " << data.ny() << "\n\n"; + + exit( 1 ); + } + + int nmasked=0, nkeep=0; + + for (int x=0; x values; + vector count; + for (int x=0; x::iterator vi; + vi = find(values.begin(), values.end(), v); + if (vi == values.end()) { + values.push_back(v); + count.push_back(1); + } else { + int ii = vi - values.begin(); + count[ii] = count[ii] + 1; + } + } + } + for (size_t i=0; i &mvd, + BoolCalc &calc) +{ + _hasUnion = calc.has_union(); + + // + // set the BoolPlane values using the mvd content + // + + BoolPlane * simple_plane = new BoolPlane [n_files]; + BoolPlane * merge_plane = new BoolPlane [n_files]; + + for (int j=0; jobjects_from_arrays(do_clusters, true, simple_plane[j]); + mvd[j]->objects_from_arrays(do_clusters, false, merge_plane[j]); + } + + // + // combine the objects into super-objects + // + const int nx = simple_plane[0].nx(); + const int ny = simple_plane[0].ny(); + + BoolPlane merge_result; // local, not used + _simple_result.set_size(nx, ny); + merge_result.set_size(nx, ny); + + string simple_name, merge_name; + + if (isFcst) { + simple_name = "Fcst_Simple"; + merge_name = "Fcst_Merge"; + } else { + simple_name = "Obs_Simple"; + merge_name = "Obs_Merge"; + } + combine_boolplanes(simple_name, simple_plane, n_files, calc, _simple_result); + combine_boolplanes(merge_name, merge_plane, n_files, calc, merge_result); + + // create ShapeData objects using something from mvd as a template + // (shape data has 1's or bad) + + _simple_sd = ShapeData(*(mvd[0]->_simple->_sd)); + for (int x=0; x_simple->_sd)); + for (int x=0; x_sd->data); +} + + +void ModeSuperObject::mask_data_super(const string &name, const MultiVarData &mvd) +{ + int nx = mvd._nx; + int ny = mvd._ny; + _mask_super(name, nx, ny, _simple_sd.data); +} diff --git a/src/tools/core/mode/mode_superobject.h b/src/tools/core/mode/mode_superobject.h new file mode 100644 index 0000000000..83278b20a6 --- /dev/null +++ b/src/tools/core/mode/mode_superobject.h @@ -0,0 +1,51 @@ +// ** Copyright UCAR (c) 1992 - 2023 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + + +#ifndef __MODE_SUPEROBJECT_H__ +#define __MODE_SUPEROBJECT_H__ + + +//////////////////////////////////////////////////////////////////////// + + +#include "combine_boolplanes.h" +#include "multivar_data.h" +#include "shapedata.h" +#include +#include + +class MultiVarData; + +class ModeSuperObject { + + private: + + public: + + ModeSuperObject(bool isFcst, int n_files, bool do_clusters, + const vector &mvd, + BoolCalc &calc); + inline ~ModeSuperObject() {} + + void mask_data_simple(const string &name, MultiVarData &mvd) const; + void mask_data_super(const string &name, const MultiVarData &mvd); + + bool _isFcst; + bool _hasUnion; + BoolPlane _simple_result; + ShapeData _simple_sd; + ShapeData _merge_sd_split; +}; + + +#endif /* __MODE_SUPEROBJECT_H__ */ + +///////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/mode/multivar_data.cc b/src/tools/core/mode/multivar_data.cc index 1d38d270ba..3cde9ccf83 100644 --- a/src/tools/core/mode/multivar_data.cc +++ b/src/tools/core/mode/multivar_data.cc @@ -142,8 +142,7 @@ MultiVarData::MultiVarData() : _merge(0), _name("notset"), _nx(0), _ny(0), - _grid(0), - _type(FileType_None) + _grid(0) { } @@ -152,23 +151,10 @@ MultiVarData::~MultiVarData() _clear(); } -void MultiVarData::checkFileTypeConsistency(const MultiVarData &mvdi, int j) -{ - bool err = false; - if (_type != mvdi._type) { - mlog << Error << "MultivarData::checkFileTypeConsistgency() -> " - << "inputs of different file types not supported " - << "Input 0:" << grdfiletype_to_string(_type).c_str() - << "Input " << j << ":" << grdfiletype_to_string(mvdi._type).c_str() - << "\n\n"; - exit ( 1 ); - } -} - void MultiVarData::init(ModeDataType dataType, const string &name, - const Grid &grid, GrdFileType type, - const string &units, + const Grid &grid, + const string &units, const string &level, double data_min, double data_max) { @@ -179,7 +165,6 @@ void MultiVarData::init(ModeDataType dataType, _nx = grid.nx(); _ny = grid.ny(); _grid = new Grid(grid); - _type = type; _units = units; _level = level; _data_min = data_min; @@ -286,7 +271,6 @@ void MultiVarData::_clear() _grid = 0; } _nx = _ny = 0; - _type = FileType_None; } diff --git a/src/tools/core/mode/multivar_data.h b/src/tools/core/mode/multivar_data.h index 142e1aa65b..f93b7f9ac5 100644 --- a/src/tools/core/mode/multivar_data.h +++ b/src/tools/core/mode/multivar_data.h @@ -85,13 +85,11 @@ class MultiVarData { MultiVarData(); ~MultiVarData(); - void checkFileTypeConsistency(const MultiVarData &mvdi, int j); - void init(ModeDataType dataType, const string &name, - const Grid &grid, GrdFileType type, + const Grid &grid, const string &units, - const string &level, + const string &level, double data_min, double data_max); void set_obj(ShapeData *sd, bool simple); @@ -109,7 +107,6 @@ class MultiVarData { string _name; int _nx, _ny; Grid *_grid; - GrdFileType _type; string _units; string _level; double _data_min, _data_max; diff --git a/src/tools/core/mode/multivar_frontend.cc b/src/tools/core/mode/multivar_frontend.cc index 2205fc4329..6bd3a5aa57 100644 --- a/src/tools/core/mode/multivar_frontend.cc +++ b/src/tools/core/mode/multivar_frontend.cc @@ -6,170 +6,88 @@ // ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA // *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* - -//////////////////////////////////////////////////////////////////////// - - -// for multivar mode, this is the default file -static const char mode_default_config [] = "MET_BASE/config/MODEMultivarConfig_default"; - -static const int dir_creation_mode = 0755; - //////////////////////////////////////////////////////////////////////// using namespace std; -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include "vx_util.h" -#include "file_exists.h" -#include "two_d_array.h" -#include "get_filenames.h" -#include "mode_conf_info.h" -#include "shapedata.h" -#include "interest.h" -#include "met_file.h" -#include "mode_usage.h" -#include "mode_exec.h" - -#include "combine_boolplanes.h" -#include "objects_from_netcdf.h" -#include "parse_file_list.h" -#include "mode_frontend.h" -#include "multivar_data.h" -#include "mode_data_type.h" +#include "multivar_frontend.h" -using namespace netCDF; +#include "mode_usage.h" +#ifdef WITH_PYTHON +#include "global_python.h" +#endif //////////////////////////////////////////////////////////////////////// extern const char * const program_name; - static const char sep [] = "===================================================="; -static const char tab [] = " "; - -// this is hardwired for the multivar case, at least for now - -static const bool do_clusters = false; +static string outdir; +static int compress_level = -1; -static string default_out_dir = "."; +// for multivar mode, this is the default file +static const char mode_default_config [] = "MET_BASE/config/MODEMultivarConfig_default"; -static ModeConfInfo config; +static const int dir_creation_mode = 0755; -static string mode_path; -static string fcst_fof; -static string obs_fof; -static string config_file; -static string outdir; +static ModeExecutive *mode_exec = 0; -static Grid verification_grid; //////////////////////////////////////////////////////////////////////// - -static void set_outdir (const StringArray &); -static void set_logfile (const StringArray &); -static void set_verbosity (const StringArray &); - -static void multivar_consistency_checks(StringArray &fcst_filenames, StringArray &obs_filenames, - BoolCalc &f_calc, BoolCalc &o_calc, int &n_fcst_files, - int &n_obs_files); - -static ConcatString set_multivar_dir(); - -static void create_verification_grid(const string &fcst_filename, - const string &obs_filename, - const ConcatString &dir); - -static MultiVarData *create_simple_objects(ModeDataType dtype, int j, int n_files, - const string &filename, - const ConcatString &dir); - -static void create_superobjects(int n_fcst_files, const vector &mvdFcst, - int n_obs_files, const vector &mvdObs, - BoolCalc &f_calc, BoolCalc &o_calc, - BoolPlane &f_simple_result, BoolPlane &o_simple_result, - ShapeData &f_simple_sd, ShapeData &o_simple_sd, - ShapeData &f_merge_sd_split, ShapeData &o_merge_sd_split); - -static void create_intensity_comparisons(int findex, int oindex, const BoolPlane &f_result, - BoolPlane &o_result, - const ConcatString &dir, - MultiVarData &mvdf, MultiVarData &mvdo, - bool has_union_f, bool has_union_o, - const string &fcst_filename, const string &obs_filename, - ShapeData &merge_f, ShapeData &merge_o); - -static void process_superobjects(ShapeData &f_result, ShapeData &o_result, - ShapeData &f_merge, ShapeData &o_merge, - int nx, int ny, const ConcatString &dir, - GrdFileType ftype, GrdFileType otype, const Grid &grid, bool has_union); - -static void mask_data(const string &name, int nx, int ny, const BoolPlane &mask, DataPlane &data); -static void mask_data_super(const string &name, int nx, int ny, DataPlane &data); - -static void read_config(const string & filename); - -static void process_command_line(const StringArray &); - -static int _mkdir(const char *dir); - -static void _debug_shape_examine(string &name, const ShapeData &sd, int nx, int ny); - +MultivarFrontEnd::MultivarFrontEnd() +{ + // this is hardwired for the multivar case, at least for now + do_clusters = false; + default_out_dir = "."; + compress_level = -1; + mode_exec = 0; +} //////////////////////////////////////////////////////////////////////// - -int multivar_frontend(const StringArray & Argv) +int MultivarFrontEnd::run(const StringArray & Argv) { - const int Argc = Argv.n(); + // initialize - if ( Argc < 4 ) multivar_usage(); - - int j, n_fcst_files, n_obs_files; - StringArray fcst_filenames; - StringArray obs_filenames; - BoolCalc f_calc, o_calc ; + init(Argv); - process_command_line(Argv); + mlog << Debug(2) << "\n" << sep << "\n"; - read_config(config_file); + // read in all the data - multivar_consistency_checks(fcst_filenames, obs_filenames, f_calc, o_calc, - n_fcst_files, n_obs_files); + // in the conf object, shift *can* be set independently for obs and fcst + int shift = config.shift_right; - bool f_has_union = f_calc.has_union(); - bool o_has_union = o_calc.has_union(); + for (int i=0; i mvdObs, mvdFcst; - - for (j=0; j 0) { - mvdFcst[0]->checkFileTypeConsistency(*mvdi, j); - } + << "\n" << sep << "\ncreating simple forecast objects from forecast " + << (j + 1) << " of " << n_fcst_files << "\n" << sep << "\n"; + MultiVarData *mvdi = create_simple_objects(ModeDataType_MvMode_Fcst, j, + n_fcst_files, fcst_filenames[j], + fcstInput[j]); mvdFcst.push_back(mvdi); mvdi->print(); } // for j - for (j=0; j 0) { - mvdObs[0]->checkFileTypeConsistency(*mvdi, j); - } + << "\n" << sep << "\ncreating simple obs objects from obs " + << (j + 1) << " of " << n_obs_files << "\n" << sep << "\n"; + MultiVarData *mvdi = create_simple_objects(ModeDataType_MvMode_Obs, j, + n_obs_files, obs_filenames[j], + obsInput[j]); mvdObs.push_back(mvdi); mvdi->print(); } // for j @@ -207,72 +123,86 @@ int multivar_frontend(const StringArray & Argv) // now create forecast and obs superobjects - BoolPlane f_simple_result, o_simple_result; - ShapeData f_simple_sd, o_simple_sd, f_merge_sd_split, o_merge_sd_split; - - create_superobjects(n_fcst_files, mvdFcst, n_obs_files, mvdObs, - f_calc, o_calc, f_simple_result, o_simple_result, - f_simple_sd, o_simple_sd, - f_merge_sd_split, o_merge_sd_split); - + ModeSuperObject fsuper(true, n_fcst_files, do_clusters, mvdFcst, f_calc); + ModeSuperObject osuper(true, n_obs_files, do_clusters, mvdObs, o_calc); + // - // Filter the data to within the superobjects only and do statistics by invoking mode algorithm again - // on the masked data pairs + // Filter the data to within the superobjects only and do statistics by invoking mode + // algorithm again on the masked data pairs // + for (int k=0; k_nx; - int ny = mvdFcst[0]->_ny; - GrdFileType ftype = mvdFcst[0]->_type; - GrdFileType otype = mvdObs[0]->_type; - Grid grid = *(mvdFcst[0]->_grid); + if (config.fcst_multivar_compare_index.n() <= 0) { - // here run one more time using superobjects as input + process_superobjects(fsuper, osuper, *mvdFcst[0], *mvdObs[0]); + } + + // + // done + // + return (0); +} - bool has_union = f_calc.has_union() || o_calc.has_union(); +//////////////////////////////////////////////////////////////////////// - process_superobjects(f_simple_sd, o_simple_sd, f_merge_sd_split, o_merge_sd_split, - nx, ny, dir, ftype, otype, grid, has_union); +MultivarFrontEnd::~MultivarFrontEnd() +{ + if ( mode_exec ) { + delete mode_exec; mode_exec = 0; } - - // free up memory - for (j=0; jgrid(); + GrdFileType ft = f->file_type(); + + //? + f->set_shift_right(shift); + + // update config now that we know file type (this sets Fcst to index i) + DataPlane dp; + + if (type == ModeDataType_MvMode_Fcst) { + config.process_config_field(ft, other_t, type, index); + f->data_plane(*(config.Fcst->var_info), dp); + fcstInput.push_back(ModeInputData(name, dp, g)); + } else { + config.process_config_field(other_t, ft, type, index); + f->data_plane(*(config.Obs->var_info), dp); + obsInput.push_back(ModeInputData(name, dp, g)); + } + + delete f; +} + +//////////////////////////////////////////////////////////////////////// - ConcatString path; +void MultivarFrontEnd::create_verif_grid() +{ + mlog << Debug(2) << "\n creating the verification grid \n" << sep << "\n"; - path = replace_path(mode_default_config); + _init_exec(ModeExecutive::TRADITIONAL, "None", "None"); + mode_exec->setup_verification_grid(fcstInput[0], obsInput[0], config); + verification_grid = mode_exec->grid; + delete mode_exec; mode_exec = 0; +} - config.read_config(path.c_str(), filename.c_str()); +//////////////////////////////////////////////////////////////////////// - return; +MultiVarData *MultivarFrontEnd::create_simple_objects(ModeDataType dtype, int j, + int n_files, + const string &filename, + const ModeInputData &input) +{ + // + // create simple non merged objects + // + _simple_objects(ModeExecutive::MULTIVAR_SIMPLE, dtype, j, n_files, + filename, input); + MultiVarData *mvdi = mode_exec->get_multivar_data(dtype); + delete mode_exec; mode_exec = 0; + // + // create simple merged objects + // + _simple_objects(ModeExecutive::MULTIVAR_SIMPLE_MERGE, dtype, j, n_files, + filename, input); + mode_exec->add_multivar_merge_data(mvdi, dtype); + delete mode_exec; mode_exec = 0; + return mvdi; } +//////////////////////////////////////////////////////////////////////// + +void +MultivarFrontEnd::create_intensity_comparisons(int findex, int oindex, + const ModeSuperObject &fsuper, + const ModeSuperObject &osuper, + MultiVarData &mvdf, MultiVarData &mvdo, + const string &fcst_filename, + const string &obs_filename) +{ + + // mask the input data to be valid only inside the simple super objects + fsuper.mask_data_simple("Fcst", mvdf); + osuper.mask_data_simple("Obs", mvdo); + + mlog << Debug(1) << "Running mvmode intensity comparisions \n\n"; + + _init_exec(ModeExecutive::MULTIVAR_INTENSITY, fcst_filename, obs_filename); + mode_exec->init_multivar_intensities(config); + + ModeConfInfo & conf = mode_exec->engine.conf_info; + conf.set_field_index(findex, oindex); + + // for multivar intensities, explicity set the level and units using stored values + // from pass1 + conf.Fcst->var_info->set_level_name(mvdf._level.c_str()); + conf.Fcst->var_info->set_units(mvdf._units.c_str()); + if (fsuper._hasUnion && conf.Fcst->merge_flag == MergeType_Thresh) { + mlog << Warning << "\nModeFrontEnd::multivar_intensity_comparisons() -> " + << "Logic includes union '||' along with 'merge_flag=THRESH' " + << ". This can lead to bad results\n\n"; + } + conf.Obs->var_info->set_level_name(mvdo._level.c_str()); + conf.Obs->var_info->set_units(mvdo._units.c_str()); + if (osuper._hasUnion && conf.Obs->merge_flag == MergeType_Thresh) { + mlog << Warning << "\nModeFrontEnd::multivar_intensity_comparisons() -> " + << "Logic includes union '||' along with 'merge_flag=THRESH' " + << ". This can lead to bad results\n\n"; + } + + // + // set up data access using inputs + // + mode_exec->setup_multivar_fcst_obs_data_intensities(mvdf, mvdo); + + // + // run the mode algorithm for multivar intensities + // + _intensity_compare_mode_algorithm(mvdf, mvdo, fsuper, osuper); + + delete mode_exec; mode_exec = 0; +} //////////////////////////////////////////////////////////////////////// +void MultivarFrontEnd::process_superobjects(ModeSuperObject &fsuper, + ModeSuperObject &osuper, + const MultiVarData &mvdf, + const MultiVarData &mvdo) +{ + mlog << Debug(1) << "Running superobject mode \n\n"; + + // set the data to 0 inside superobjects and missing everywhere else + + fsuper.mask_data_super("FcstSimple", mvdf); + osuper.mask_data_super("ObsSimple", mvdo); + + _init_exec(ModeExecutive::MULTIVAR_SUPER, "None", "None"); + mode_exec->init_multivar_intensities(config); + + ModeConfInfo & conf = mode_exec->engine.conf_info; + if ((fsuper._hasUnion || osuper._hasUnion) && + (conf.Fcst->merge_flag == MergeType_Thresh || + conf.Obs->merge_flag == MergeType_Thresh)) { + mlog << Warning << "\nModeFrontEnd::run_super() -> " + << "Logic includes union '||' along with 'merge_flag=THRESH' " + << ". This can lead to bad results\n\n"; + } + + // + // set up data access using inputs + // + mode_exec->setup_multivar_fcst_obs_data_super(fsuper._simple_sd, osuper._simple_sd, + *mvdf._grid); + + // run the mode algorithm + _superobject_mode_algorithm(fsuper, osuper); + + delete mode_exec; mode_exec = 0; +} + +//////////////////////////////////////////////////////////////////////// -void process_command_line(const StringArray & argv) +void MultivarFrontEnd::_process_command_line(const StringArray & argv) { @@ -357,6 +436,7 @@ void process_command_line(const StringArray & argv) cline.add(set_outdir, "-outdir", 1); cline.add(set_logfile, "-log", 1); cline.add(set_verbosity, "-v", 1); + cline.add(set_compress, "-compress", 1); cline.parse(); @@ -364,11 +444,37 @@ void process_command_line(const StringArray & argv) // should be 3 arguments left // - fcst_fof = cline[0]; obs_fof = cline[1]; config_file = cline[2]; + return; + +} + +//////////////////////////////////////////////////////////////////////// + + +void MultivarFrontEnd::_read_config(const string & filename) + +{ + + ConcatString path; + + path = replace_path(mode_default_config); + + config.read_config(path.c_str(), filename.c_str()); + + // process the config except for the fields + config.process_config_except_fields(); + + // done once here, used for all data + // what is this, command line overrides config? look deeper.. remove from exec + // except traditional mode + if (compress_level >= 0) config.nc_info.set_compress_level(compress_level); + // from within mode_exec: + // engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); + return; @@ -376,9 +482,7 @@ void process_command_line(const StringArray & argv) //////////////////////////////////////////////////////////////////////// -void multivar_consistency_checks(StringArray &fcst_filenames, StringArray &obs_filenames, - BoolCalc &f_calc, BoolCalc &o_calc, int &n_fcst_files, - int &n_obs_files) +void MultivarFrontEnd::_setup_inputs() { // // make sure the multivar logic programs are in the config file @@ -464,450 +568,191 @@ void multivar_consistency_checks(StringArray &fcst_filenames, StringArray &obs_f //////////////////////////////////////////////////////////////////////// -ConcatString set_multivar_dir() +void MultivarFrontEnd::_set_output_path() { - ConcatString dir; int status; - dir.clear(); + output_path.clear(); // no longer want numbered subdirectories - if ( outdir.length() > 0 ) dir << outdir; + if ( outdir.length() > 0 ) output_path << outdir; // // test to see of the output directory for this // mode runs exists, and if not, create it // - if ( ! directory_exists(dir.c_str()) ) { + if ( ! directory_exists(output_path.c_str()) ) { mlog << Debug(2) << program_name << ": creating output directory \"" - << dir << "\"\n\n"; + << output_path << "\"\n\n"; - status = _mkdir(dir.c_str()); + status = _mkdir(output_path.c_str()); if ( status < 0 ) { mlog << Error << "\nset_multivar_dir() ->" << " unable to create output directory \"" - << dir << "\"\n\n"; + << output_path << "\"\n\n"; exit ( 1 ); } } - return dir; } //////////////////////////////////////////////////////////////////////// -void create_verification_grid(const string &fcst_filename, const string &obs_filename, - const ConcatString &dir) +int MultivarFrontEnd::_mkdir(const char *dir) { - ConcatString command; - StringArray a, mode_argv; - - // - // build the command for running mode frontend - // - - mode_argv.clear(); - mode_argv.add(mode_path); - mode_argv.add(fcst_filename); - mode_argv.add(obs_filename); - mode_argv.add(config_file); - - command << cs_erase - << mode_path << ' ' - << fcst_filename << ' ' - << obs_filename << ' ' - << config_file; - - mode_argv.add("-v"); - char junk [256]; - snprintf(junk, sizeof(junk), "%d", mlog.verbosity_level()); - mode_argv.add(junk); - - mode_argv.add("-outdir"); - mode_argv.add(dir); - - command << " -v " << mlog.verbosity_level(); - command << " -outdir " << dir; - - // - // run the pass1 portions of mode, which creates simple objects - // - - mlog << Debug(3) << "Running mode command: \"" << command << "\"\n\n"; - ModeFrontEnd *frontend = new ModeFrontEnd; - verification_grid = frontend->create_verification_grid(mode_argv); - delete frontend; + char tmp[256]; + char *p = NULL; + size_t len; + snprintf(tmp, sizeof(tmp),"%s",dir); + len = strlen(tmp); + if (tmp[len - 1] == '/') + tmp[len - 1] = 0; + for (p = tmp + 1; *p; p++) + if (*p == '/') { + *p = 0; + string s = tmp; + if (s != ".") { + if (mkdir(tmp, dir_creation_mode) < 0) { + mlog << Error << "\n_mkdir() -> Error making " << tmp << "\n"; + return -1; + } + } + *p = '/'; + } + return (mkdir(tmp, dir_creation_mode)); } //////////////////////////////////////////////////////////////////////// -MultiVarData *create_simple_objects(ModeDataType dtype, int j, int n_files, - const string &filename, - const ConcatString &dir) +void MultivarFrontEnd::_simple_objects(ModeExecutive::Processing_t p, + ModeDataType dtype, + int j, int n_files, const string &filename, + const ModeInputData &input) { - ConcatString command; - StringArray a, mode_argv; - - // - // build the command for running mode frontend - // - - mode_argv.clear(); - mode_argv.add(mode_path); - mode_argv.add(filename); - mode_argv.add(config_file); - - command << cs_erase - << mode_path << ' ' - << filename << ' ' - << config_file; - - mode_argv.add("-v"); - char junk [256]; - snprintf(junk, sizeof(junk), "%d", mlog.verbosity_level()); - mode_argv.add(junk); - - mode_argv.add("-outdir"); - mode_argv.add(dir); - - command << " -v " << mlog.verbosity_level(); - command << " -outdir " << dir; - - // - // create the simple objects, forecast or obs, from this input data file - // - - mlog << Debug(3) << "Running mode command: \"" << command << "\"\n\n"; - ModeFrontEnd *frontend = new ModeFrontEnd; - int status = frontend->create_multivar_simple_objects(mode_argv, dtype, verification_grid, j, n_files); - MultiVarData *mvdi = frontend->get_multivar_data(dtype); - delete frontend; - - // - // create simple merge objects - // - - frontend = new ModeFrontEnd; - status = frontend->create_multivar_merge_objects(mode_argv, dtype, verification_grid, j, n_files); - - // add the merge results to the mvdi object - frontend->add_multivar_merge_data(mvdi, dtype); - delete frontend; - - return mvdi; -} + if (dtype == ModeDataType_MvMode_Fcst) { + _init_exec(p, filename, "None"); + mode_exec->init_multivar_simple(j, n_files, dtype, config); + mode_exec->setup_multivar_fcst_data(verification_grid, input); + } else { + _init_exec(p, "None", filename); + mode_exec->init_multivar_simple(j, n_files, dtype, config); + mode_exec->setup_multivar_obs_data(verification_grid, input); + } + + _simple_mode_algorithm(p); +} //////////////////////////////////////////////////////////////////////// -void create_superobjects(int n_fcst_files, const vector &mvdFcst, - int n_obs_files, const vector &mvdObs, - BoolCalc &f_calc, BoolCalc &o_calc, - BoolPlane &f_simple_result, BoolPlane &o_simple_result, - ShapeData &f_simple_sd, ShapeData &o_simple_sd, - ShapeData &f_merge_sd_split, ShapeData &o_merge_sd_split) +void MultivarFrontEnd::_init_exec(ModeExecutive::Processing_t p, + const string &ffile, + const string &ofile) { - // - // set the BoolPlane values using the mvd content - // + mlog << Debug(1) << "Running multivar front end for " << ModeExecutive::stype(p) << "\n"; - BoolPlane * f_simple_plane = new BoolPlane [n_fcst_files]; - BoolPlane * o_simple_plane = new BoolPlane [n_obs_files]; - BoolPlane * f_merge_plane = new BoolPlane [n_fcst_files]; - BoolPlane * o_merge_plane = new BoolPlane [n_obs_files]; + if ( mode_exec ) { delete mode_exec; mode_exec = 0; } - for (int j=0; jobjects_from_arrays(do_clusters, true, f_simple_plane[j]); - mvdFcst[j]->objects_from_arrays(do_clusters, false, f_merge_plane[j]); - } - - for (int j=0; jobjects_from_arrays(do_clusters, true, o_simple_plane[j]); - mvdObs[j]->objects_from_arrays(do_clusters, false, o_merge_plane[j]); - } + mode_exec = new ModeExecutive(); + // compress_level = -1; + mode_exec->fcst_file = ffile; + mode_exec->obs_file = ofile; - // - // combine the objects into super-objects - // - const int nx = f_simple_plane[0].nx(); - const int ny = f_simple_plane[0].ny(); - - BoolPlane f_merge_result, o_merge_result; - f_simple_result.set_size(nx, ny); - o_simple_result.set_size(nx, ny); - f_merge_result.set_size(nx, ny); - o_merge_result.set_size(nx, ny); - - combine_boolplanes("Fcst_Simple", f_simple_plane, n_fcst_files, f_calc, f_simple_result); - combine_boolplanes("Obs_Simple", o_simple_plane, n_obs_files, o_calc, o_simple_result); - combine_boolplanes("Fcst_Merge", f_merge_plane, n_fcst_files, f_calc, f_merge_result); - combine_boolplanes("Obs_Merge", o_merge_plane, n_obs_files, o_calc, o_merge_result); - - - // create ShapeData objects using something from mvd as a template - // (shape data has 1's or bad) - - f_simple_sd = ShapeData(*(mvdFcst[0]->_simple->_sd)); - for (int x=0; x_simple->_sd)); - for (int x=0; x_simple->_sd)); - for (int x=0; x_simple->_sd)); - for (int x=0; xmatch_config_file = config_file; // this is never used + mode_exec->out_dir = output_path; } - //////////////////////////////////////////////////////////////////////// void -create_intensity_comparisons(int findex, int oindex, const BoolPlane &f_result, - BoolPlane &o_result, const ConcatString &dir, - MultiVarData &mvdf, MultiVarData &mvdo, - bool has_union_f, bool has_union_o, - const string &fcst_filename, const string &obs_filename, - ShapeData &merge_f, ShapeData &merge_o) +MultivarFrontEnd::_superobject_mode_algorithm(const ModeSuperObject &fsuper, + const ModeSuperObject &osuper) { - - // mask the input data to be valid only inside the simple super objects - int nx = mvdf._nx; - int ny = mvdf._ny; - - mask_data("Fcst", nx, ny, f_result, mvdf._simple->_sd->data); - mask_data("Obs", nx, ny, o_result, mvdo._simple->_sd->data); - - // - // build the command for running mode frontend - // - StringArray mode_argv; - char junk [256]; - - mode_argv.clear(); - mode_argv.add(mode_path); - mode_argv.add(fcst_filename); - mode_argv.add(obs_filename); - mode_argv.add(config_file); - mode_argv.add("-v"); - snprintf(junk, sizeof(junk), "%d", mlog.verbosity_level()); - mode_argv.add(junk); - mode_argv.add("-outdir"); - mode_argv.add(dir); - - mlog << Debug(1) << "Running mvmode intensity comparisions \n\n"; - - ModeFrontEnd *frontend = new ModeFrontEnd; - int status = frontend->multivar_intensity_comparisons(mode_argv, mvdf, mvdo, has_union_f, - has_union_o, merge_f, merge_o, findex, - oindex); - delete frontend; + _mode_algorithm_init(); + mode_exec->clear_internal_r_index(); + mode_exec->do_conv_thresh_multivar_super(); + mode_exec->do_match_merge_multivar(fsuper._merge_sd_split, osuper._merge_sd_split, + ModeExecutive::MULTIVAR_SUPER); + mode_exec->process_output_multivar_super(); + mode_exec->clear_internal_r_index(); +#ifdef WITH_PYTHON + GP.finalize(); + #endif } //////////////////////////////////////////////////////////////////////// -void process_superobjects(ShapeData &f_result, ShapeData &o_result, - ShapeData &f_merge, ShapeData &o_merge, - int nx, int ny, const ConcatString &dir, - GrdFileType ftype, GrdFileType otype, const Grid &grid, - bool has_union) +void +MultivarFrontEnd::_intensity_compare_mode_algorithm(const MultiVarData &mvdf, + const MultiVarData &mvdo, + const ModeSuperObject &fsuper, + const ModeSuperObject &osuper) { - StringArray mode_argv; - char junk [256]; + _mode_algorithm_init(); + mode_exec->do_conv_thresh_multivar_intensity_compare(); + mode_exec->do_match_merge_multivar(fsuper._merge_sd_split, osuper._merge_sd_split, + ModeExecutive::MULTIVAR_INTENSITY); + // here replace raw data and min/max for plotting + mode_exec->process_output_multivar_intensity_compare(&mvdf, &mvdo); + mode_exec->clear_internal_r_index(); +#ifdef WITH_PYTHON + GP.finalize(); + #endif +} - // - // build the command for running mode frontend - // - mode_argv.clear(); - mode_argv.add(mode_path); - mode_argv.add(config_file); - mode_argv.add("-v"); - snprintf(junk, sizeof(junk), "%d", mlog.verbosity_level()); - mode_argv.add(junk); - mode_argv.add("-outdir"); - mode_argv.add(dir); - - mlog << Debug(1) << "Running superobject mode \n\n"; - - // set the data to 0 inside superobjects and missing everywhere else - mask_data_super("FcstSimple", nx, ny, f_result.data); - mask_data_super("ObsSimple", nx, ny, o_result.data); - - - ModeFrontEnd *frontend = new ModeFrontEnd; - int status = frontend->run_super(mode_argv, f_result, o_result, - f_merge, o_merge, ftype, otype, grid, has_union); - delete frontend; -} - //////////////////////////////////////////////////////////////////////// -void mask_data(const string &name, int nx, int ny, const BoolPlane &bp, DataPlane &data) +void MultivarFrontEnd::_simple_mode_algorithm(ModeExecutive::Processing_t p) { - - if (nx != data.nx() || ny != data.ny()) { - mlog << Error << "\nmask_data() -> " << name - << " :dimensions don't match " << nx << " " << ny - << " " << data.nx() << " " << data.ny() << "\n\n"; - - exit( 1 ); - } - - int nmasked=0, nkeep=0; + _mode_algorithm_init(); + mode_exec->clear_internal_r_index(); + mode_exec->do_conv_thresh_multivar_simple(p); + mode_exec->clear_internal_r_index(); - for (int x=0; xengine.conf_info; + if ( conf.quilt ) { + mlog << Error << "\nMultiVarFontend::mode_algorithm() -> " + << "quilting not yet implemented for multivar mode \n\n"; + exit ( 1 ); } - - mlog << Debug(1) << name << " superobject masking.." - << nkeep << " points of " - << nmasked + nkeep << " in superobjects\n"; -} + int NCT = conf.n_conv_threshs(); + int NCR = conf.n_conv_radii(); + if ( NCT != NCR ) { -//////////////////////////////////////////////////////////////////////// -void mask_data_super(const string &name, int nx, int ny, DataPlane &data) -{ + mlog << Error << "\nMultivarFrontEnd::_mode_algorithm_init() ->" + << "all convolution radius and threshold arrays must have the same number of elements\n\n"; - if (nx != data.nx() || ny != data.ny()) { - mlog << Error << "\nmask_data_super() -> " << name - << " :dimensions don't match " << nx << " " << ny - << " " << data.nx() << " " << data.ny() << "\n\n"; + exit ( 1 ); - exit( 1 ); } - int nmasked=0, nkeep=0; - - for (int x=0; x 1) { - for (int y=0; y" + << ": multiple convolution radii and thresholds not implemented in multivar mode\n\n"; - if(is_bad_data(data.get(x,y))) { - nmasked ++; - } else { - data.set(0.0, x, y); - nkeep ++; - } - } + exit ( 1 ); } - - mlog << Debug(1) << name << " superobject masking.." - << nkeep << " points of " - << nmasked + nkeep << " in superobjects\n"; } -//////////////////////////////////////////////////////////////////////// - -int _mkdir(const char *dir) -{ - char tmp[256]; - char *p = NULL; - size_t len; - - snprintf(tmp, sizeof(tmp),"%s",dir); - len = strlen(tmp); - if (tmp[len - 1] == '/') - tmp[len - 1] = 0; - for (p = tmp + 1; *p; p++) - if (*p == '/') { - *p = 0; - string s = tmp; - if (s != ".") { - if (mkdir(tmp, dir_creation_mode) < 0) { - mlog << Error << "\n_mkdir() -> Error making " << tmp << "\n"; - return -1; - } - } - *p = '/'; - } - - return (mkdir(tmp, dir_creation_mode)); -} -//////////////////////////////////////////////////////////////////////// -void _debug_shape_examine(string &name, const ShapeData &sd, int nx, int ny) -{ - vector values; - vector count; - for (int x=0; x::iterator vi; - vi = find(values.begin(), values.end(), v); - if (vi == values.end()) { - values.push_back(v); - count.push_back(1); - } else { - int ii = vi - values.begin(); - count[ii] = count[ii] + 1; - } - } - } - for (size_t i=0; i +#include "mode_conf_info.h" +#include "two_d_array.h" +#include "bool_calc.h" +#include "multivar_data.h" +#include "mode_superobject.h" +#include "mode_input_data.h" +#include "mode_exec.h" + +class MultivarFrontEnd { + +private: + + int n_fcst_files, n_obs_files; + StringArray fcst_filenames; + StringArray obs_filenames; + BoolCalc f_calc, o_calc ; + vector fcstInput, obsInput; + vector mvdFcst, mvdObs; + string fcst_fof; + string obs_fof; + + void _process_command_line(const StringArray &); + void _read_config(const string & filename); + void _setup_inputs(); + void _set_output_path(); + int _mkdir(const char *dir); + void _simple_objects(ModeExecutive::Processing_t p, ModeDataType dtype, + int j, int n_files, const string &filename, + const ModeInputData &input); + void _init_exec(ModeExecutive::Processing_t p, const string &ffile, const string &ofile); + void _superobject_mode_algorithm(const ModeSuperObject &fsuper, const ModeSuperObject &osuper); + void _intensity_compare_mode_algorithm(const MultiVarData &mvdf, const MultiVarData &mvdo, + const ModeSuperObject &fsuper, const ModeSuperObject &osuper); + void _simple_mode_algorithm(ModeExecutive::Processing_t p); + void _mode_algorithm_init() const; + +public: + + bool do_clusters; + string default_out_dir; + ModeConfInfo config; + ConcatString output_path; + string mode_path; + string config_file; + Grid verification_grid; + + MultivarFrontEnd(); + + ~MultivarFrontEnd(); + + + int run(const StringArray & Argv); + void init(const StringArray & Argv); + + static void set_outdir (const StringArray &); + static void set_logfile (const StringArray &); + static void set_verbosity (const StringArray &); + static void set_compress (const StringArray &); + + void read_input(const string &name, int index, ModeDataType type, + GrdFileType f_t, GrdFileType other_t, int shift); + + + void create_verif_grid(void); + + MultiVarData *create_simple_objects(ModeDataType dtype, int j, int n_files, + const string &filename, + const ModeInputData &input); + + void create_intensity_comparisons(int findex, int oindex, + const ModeSuperObject &fsuper, + const ModeSuperObject &osuper, + MultiVarData &mvdf, MultiVarData &mvdo, + const string &fcst_filename, + const string &obs_filename); + + void process_superobjects(ModeSuperObject &fsuper, + ModeSuperObject &osuper, + const MultiVarData &mvdf, + const MultiVarData &mvdo); + +}; + + +#endif /* __MULTIVAR_FRONT_END_H__ */ + + +/////////////////////////////////////////////////////////////////////////