From f1a345ec0befbbf178f420376e67eb38a06eff6c Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Wed, 20 Dec 2023 23:00:16 +0000 Subject: [PATCH 01/14] Working mvmode with percentile thresholds and grid_type fixes, plus more unit tests --- .../scripts/environment/development.seneca | 45 +- internal/test_unit/config/MODEConfig_multivar | 274 ++++++++ .../test_unit/config/MODEConfig_multivar_3_2 | 275 ++++++++ .../config/MODEConfig_multivar_super | 271 ++++++++ internal/test_unit/xml/unit_mode_multivar.xml | 85 ++- src/basic/vx_config/config_file.cc | 14 + src/libcode/vx_shapedata/Makefile.am | 1 + src/libcode/vx_shapedata/engine.cc | 9 +- src/libcode/vx_shapedata/mode_conf_info.cc | 596 ++++++++++++++---- src/libcode/vx_shapedata/mode_conf_info.h | 42 +- src/libcode/vx_shapedata/mode_field_info.cc | 38 +- src/libcode/vx_shapedata/mode_field_info.h | 8 +- src/libcode/vx_shapedata/mode_input_data.h | 45 ++ src/tools/core/mode/Makefile.am | 1 + src/tools/core/mode/combine_boolplanes.h | 3 +- src/tools/core/mode/mode.cc | 2 +- src/tools/core/mode/mode_exec.cc | 284 ++------- src/tools/core/mode/mode_exec.h | 25 +- src/tools/core/mode/mode_frontend.cc | 180 +++--- src/tools/core/mode/mode_frontend.h | 49 +- src/tools/core/mode/mode_superobject.cc | 102 +++ src/tools/core/mode/mode_superobject.h | 45 ++ src/tools/core/mode/multivar_frontend.cc | 467 +++++--------- 23 files changed, 2029 insertions(+), 832 deletions(-) create mode 100644 internal/test_unit/config/MODEConfig_multivar create mode 100644 internal/test_unit/config/MODEConfig_multivar_3_2 create mode 100644 internal/test_unit/config/MODEConfig_multivar_super create mode 100644 src/libcode/vx_shapedata/mode_input_data.h create mode 100644 src/tools/core/mode/mode_superobject.cc create mode 100644 src/tools/core/mode/mode_superobject.h diff --git a/internal/scripts/environment/development.seneca b/internal/scripts/environment/development.seneca index cc623c8d95..5c69a32e9e 100644 --- a/internal/scripts/environment/development.seneca +++ b/internal/scripts/environment/development.seneca @@ -1,34 +1,35 @@ # Define the development environment for NCAR project machine seneca -# Based on settings in /usr/local/src/met/README.snat +# Based on settings in /nrit/ral/src/met/README.ral-it # Top-level MET project directory MET_PROJ_DIR=/d1/projects/MET # Variables required to build MET export MET_DEVELOPMENT=true -export MET_DST=/usr/local +export MET_DST=/nrit/ral export MET_PROJ=${MET_DST}/proj-9.2.1 -export MET_NETCDF=${MET_DST}/netcdf-4.7.0/gcc-8.3.0 -export MET_HDF5=${MET_DST}/hdf5-1.8.21 -export MET_HDFINC=${MET_DST}/hdf4-4.2.15/include/hdf -export MET_HDFLIB=${MET_DST}/hdf4-4.2.15/lib -export MET_HDFEOS=${MET_DST}/hdf-eos2-20v1 -export MET_BUFR=${MET_DST} -export MET_GRIB2C=${MET_DST} -export MET_GSL=${MET_PROJ_DIR}/MET_releases/external_libs/gnu_8.3.0 +export MET_NETCDF=${MET_DST}/netcdf-4.9.2/gnu-12.2.0 +export MET_HDF5=${MET_DST}/hdf5-1.14.2 +export MET_HDFINC=${MET_DST}/hdf4-4.2.16-2/include/hdf +export MET_HDFLIB=${MET_DST}/hdf4-4.2.16-2/lib +export MET_HDFEOS=${MET_DST}/hdf-eos2-3.0 +export MET_BUFR=${MET_DST}/nceplibs-1.4.0/bufr-12.0.1 +export MET_GRIB2C=${MET_DST}/nceplibs-1.4.0/g2c-1.6.2 +export MET_GSLINC=/usr/include +export MET_GSLLIB=/usr/lib/x86_64-linux-gnu export MET_CAIROINC=/usr/include/cairo export MET_CAIROLIB=/usr/lib/x86_64-linux-gnu export MET_FREETYPEINC=/usr/include/freetype2 export MET_FREETYPELIB=/usr/lib/x86_64-linux-gnu -export MET_JASPER=${MET_DST}/jasper-1.900.1 -export MET_ATLAS=${MET_PROJ_DIR}/MET_releases/external_libs/gnu_8.3.0/atlas -export MET_ECKIT=${MET_PROJ_DIR}/MET_releases/external_libs/gnu_8.3.0/eckit +export MET_ATLAS=${MET_DST}/atlas-0.30.0 +export MET_ECKIT=${MET_DST}/eckit-1.20.2 +export JASPER=${MET_DST}/jasper-1.900.1 # For Python 3 in met-9.0 -export MET_PYTHON=/usr/local/met-python3 -export MET_PYTHON_BIN_EXE=${MET_PYTHON}/bin/python3.8 -export MET_PYTHON_CC="-I${MET_PYTHON}/include/python3.8" -export MET_PYTHON_LD="-L${MET_PYTHON}/lib -lpython3.8 -lcrypt -lpthread -ldl -lutil -lm" +export MET_PYTHON=${MET_DST}/met-python3 +export MET_PYTHON_BIN_EXE=${MET_PYTHON}/bin/python3.10 +export MET_PYTHON_CC="-I${MET_PYTHON}/include/python3.10" +export MET_PYTHON_LD="-L${MET_PYTHON}/lib -lpython3.10 -lcrypt -lpthread -ldl -lutil -lm" # -D__64BIT__ is required because we've compiled libgrib2c.a with that flag export CFLAGS="-DUNDERSCORE -fPIC -D__64BIT__ -g" @@ -37,21 +38,21 @@ export CXXFLAGS=${CFLAGS} # Set LDFLAGS to include -rpath settings when compiling MET export LDFLAGS="-Wl,--disable-new-dtags" export LDFLAGS="${LDFLAGS} -Wl,-rpath,${MET_DST}/lib:${MET_HDFEOS}/lib:${MET_PROJ}/lib:${MET_NETCDF}/lib:${MET_DST}/zlib-1.2.11/lib:${MET_DST}/szip-2.1.1/lib" -export LDFLAGS="${LDFLAGS} -Wl,-rpath,${MET_HDFLIB}:${MET_HDF5}/lib:${MET_GSL}/lib:${MET_PYTHON}/lib:${MET_JASPER}/lib" -export LDFLAGS="${LDFLAGS} -L${MET_JASPER}/lib" +export LDFLAGS="${LDFLAGS} -Wl,-rpath,${MET_HDFLIB}:${MET_HDF5}/lib:${MET_GSL}/lib:${MET_PYTHON}/lib:${JASPER}/lib" +export LDFLAGS="${LDFLAGS} -L${JASPER}/lib" # Variables required to run MET export MET_TEST_INPUT=${MET_PROJ_DIR}/MET_test_data/unit_test export MET_FONT_DIR=${MET_TEST_INPUT}/fonts # Define Rscript to use a version with the ncdf4 package 1.17 or later -export MET_TEST_RSCRIPT=/usr/local/R-4.1.2/bin/Rscript +export MET_TEST_RSCRIPT=/nrit/ral/R-4.3.1/bin/Rscript # This is a cron script -- create the shell environment for this job # - NCO is for ncdiff # - NetCDF is for ncdump. -export PATH="/usr/local/nco/bin:/usr/local/netcdf/bin:\ - /usr/local/sbin:/usr/local/bin:/usr/sbin:\ +export PATH="/nrit/ral/nco/bin:/nrit/ral/netcdf4/bin:\ + /nrit/ral/sbin:/nrit/ral/bin:/usr/sbin:\ /usr/bin:/sbin:/bin:/usr/bin/X11:/opt/bin:$PATH" # SonarQube diff --git a/internal/test_unit/config/MODEConfig_multivar b/internal/test_unit/config/MODEConfig_multivar new file mode 100644 index 0000000000..d3fba7556d --- /dev/null +++ b/internal/test_unit/config/MODEConfig_multivar @@ -0,0 +1,274 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// MODE configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +// model = +model = "HRRR"; + +// +// Output description to be written +// +// desc = + +// +// Output observation type to be written +// +// obtype = +obtype = "ANALYSIS"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = {to_grid = FCST;method = NEAREST;width = 1;vld_thresh = 0.5;} + +//////////////////////////////////////////////////////////////////////////////// + +// +`// Approximate grid resolution (km) +// +// grid_res = +// + +//////////////////////////////////////////////////////////////////////////////// + +// +// Run all permutations of radius and threshold +// +// quilt = +// ${METPLUS_QUILT} + +// +// MODE Multivar boolean combination logic +// +//multivar_logic = +//${METPLUS_MULTIVAR_LOGIC} +multivar_logic = "#1 && #2 && #3"; + +// keep this around to compare to older versions, the new version doesn't read this +multivar_intensity_flag = [FALSE, TRUE, TRUE]; + +multivar_intensity_compare_fcst = [ 2, 3 ]; +multivar_intensity_compare_obs = [ 2, 3 ]; + +// +// Forecast and observation fields to be verified +// +fcst = { + //${METPLUS_FCST_FIELD} + field = [{ name="CSNOW"; level="L0"; conv_radius = 0; conv_thresh = ==1; file_type = GRIB2; merge_flag = NONE; }, + { name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; }, + { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; }]; + + //${METPLUS_FCST_CENSOR_THRESH} + //${METPLUS_FCST_CENSOR_VAL} + //${METPLUS_FCST_CONV_RADIUS} + //${METPLUS_FCST_CONV_THRESH} + //${METPLUS_FCST_VLD_THRESH} + //${METPLUS_FCST_FILTER_ATTR_NAME} + filter_attr_name = ["AREA"]; + + //${METPLUS_FCST_FILTER_ATTR_THRESH} + filter_attr_thresh = [>=25]; + + //${METPLUS_FCST_MERGE_THRESH} + //${METPLUS_FCST_MERGE_FLAG} + //${METPLUS_FCST_FILE_TYPE} + //file_type = GRIB2; + multivar_name = "Snow"; + multivar_level = "LO"; +} + +obs = { + //${METPLUS_OBS_FIELD} + field = [{ name="PrecipFlag"; level="L0"; conv_radius = 0; conv_thresh = ==3; file_type = GRIB2; merge_flag = NONE; }, + { name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; }, + { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; } ]; + + //${METPLUS_OBS_CENSOR_THRESH} + //${METPLUS_OBS_CENSOR_VAL} + //${METPLUS_OBS_CONV_RADIUS} + //${METPLUS_OBS_CONV_THRESH} + //${METPLUS_OBS_VLD_THRESH} + + //${METPLUS_OBS_FILTER_ATTR_NAME} + filter_attr_name = ["AREA"]; + + //${METPLUS_OBS_FILTER_ATTR_THRESH} + filter_attr_thresh = [>=25]; + + // ${METPLUS_OBS_MERGE_THRESH} + // ${METPLUS_OBS_MERGE_FLAG} + // ${METPLUS_OBS_FILE_TYPE} + multivar_name = "Precip"; + multivar_level = "LO"; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Handle missing data +// +// mask_missing_flag = +// ${METPLUS_MASK_MISSING_FLAG} + +// +// Match objects between the forecast and observation fields +// +//match_flag = +//${METPLUS_MATCH_FLAG} +match_flag = MERGE_BOTH; +// +// Maximum centroid distance for objects to be compared +// +//max_centroid_dist = +//${METPLUS_MAX_CENTROID_DIST} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +//mask = { +//${METPLUS_MASK_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fuzzy engine weights +// +//weight = { +//${METPLUS_WEIGHT_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fuzzy engine interest functions +// +interest_function = { + + //${METPLUS_INTEREST_FUNCTION_CENTROID_DIST} + + //${METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST} + + //${METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST} + + angle_diff = ( + ( 0.0, 1.0 ) + ( 30.0, 1.0 ) + ( 90.0, 0.0 ) + ); + + aspect_diff = ( + ( 0.00, 1.0 ) + ( 0.10, 1.0 ) + ( 0.75, 0.0 ) + ); + + corner = 0.8; + ratio_if = ( + ( 0.0, 0.0 ) + ( corner, 1.0 ) + ( 1.0, 1.0 ) + ); + + area_ratio = ratio_if; + + int_area_ratio = ( + ( 0.00, 0.00 ) + ( 0.10, 0.50 ) + ( 0.25, 1.00 ) + ( 1.00, 1.00 ) + ); + + curvature_ratio = ratio_if; + + complexity_ratio = ratio_if; + + inten_perc_ratio = ratio_if; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Total interest threshold for determining matches +// +//total_interest_thresh = +//${METPLUS_TOTAL_INTEREST_THRESH} + +// +// Interest threshold for printing output pair information +// +print_interest_thresh = 0.0; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Plotting information +// +met_data_dir = "MET_BASE"; + +fcst_raw_plot = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; +} + +obs_raw_plot = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; +} + +object_plot = { + color_table = "MET_BASE/colortables/mode_obj.ctable"; +} + +// +// Boolean for plotting on the region of valid data within the domain +// +plot_valid_flag = FALSE; + +// +// Plot polyline edges using great circle arcs instead of straight lines +// +plot_gcarc_flag = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// NetCDF matched pairs, PostScript, and contingency table output files +// +//ps_plot_flag = +//${METPLUS_PS_PLOT_FLAG} + +//nc_pairs_flag = { +//${METPLUS_NC_PAIRS_FLAG_DICT} + +//ct_stats_flag = +//${METPLUS_CT_STATS_FLAG} + + +//////////////////////////////////////////////////////////////////////////////// + +shift_right = 0; // grid squares + +//////////////////////////////////////////////////////////////////////////////// + +//${METPLUS_OUTPUT_PREFIX} +//version = "V10.0"; + +//tmp_dir = "${MET_TMP_DIR}"; +tmp_dir = "/tmp"; + +//////////////////////////////////////////////////////////////////////////////// + +//${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/internal/test_unit/config/MODEConfig_multivar_3_2 b/internal/test_unit/config/MODEConfig_multivar_3_2 new file mode 100644 index 0000000000..716b93c9be --- /dev/null +++ b/internal/test_unit/config/MODEConfig_multivar_3_2 @@ -0,0 +1,275 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Mode configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +// model = +model = "HRRR"; + +// +// Output description to be written +// +// desc = + +// +// Output observation type to be written +// +// obtype = +obtype = "ANALYSIS"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +// regrid = { +regrid = {to_grid = FCST;method = NEAREST;width = 1;vld_thresh = 0.5;} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Approximate grid resolution (km) +// +// grid_res = +// + +//////////////////////////////////////////////////////////////////////////////// + +// +// Run all permutations of radius and threshold +// +// quilt = +// ${METPLUS_QUILT} + +// +// MODE Multivar boolean combination logic +// +//multivar_logic = +//${METPLUS_MULTIVAR_LOGIC} +multivar_logic = "#1 && #2 && #3"; + +multivar_intensity_compare_fcst = [ 2, 2 ]; +multivar_intensity_compare_obs = [ 1, 2 ]; + +// +// Forecast and observation fields to be verified +// +fcst = { + //${METPLUS_FCST_FIELD} + field = [{ name="CSNOW"; level="L0"; conv_radius = 0; conv_thresh = ==1; file_type = GRIB2; merge_flag = NONE; }, + { name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; }, + { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; } ]; + + multivar_logic = "#1 && #2 && #3"; + + //${METPLUS_FCST_CENSOR_THRESH} + //${METPLUS_FCST_CENSOR_VAL} + //${METPLUS_FCST_CONV_RADIUS} + //${METPLUS_FCST_CONV_THRESH} + //${METPLUS_FCST_VLD_THRESH} + //${METPLUS_FCST_FILTER_ATTR_NAME} + filter_attr_name = ["AREA"]; + + //${METPLUS_FCST_FILTER_ATTR_THRESH} + filter_attr_thresh = [>=25]; + + //${METPLUS_FCST_MERGE_THRESH} + //${METPLUS_FCST_MERGE_FLAG} + //${METPLUS_FCST_FILE_TYPE} + //file_type = GRIB2; + multivar_name = "Snow"; + multivar_level = "LO"; +} + +obs = { + //${METPLUS_OBS_FIELD} + field = [{ name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; }, + { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; } ]; + + multivar_logic = "#1 && #2"; + + //${METPLUS_OBS_CENSOR_THRESH} + //${METPLUS_OBS_CENSOR_VAL} + //${METPLUS_OBS_CONV_RADIUS} + //${METPLUS_OBS_CONV_THRESH} + //${METPLUS_OBS_VLD_THRESH} + + //${METPLUS_OBS_FILTER_ATTR_NAME} + filter_attr_name = ["AREA"]; + + //${METPLUS_OBS_FILTER_ATTR_THRESH} + filter_attr_thresh = [>=25]; + + // ${METPLUS_OBS_MERGE_THRESH} + // ${METPLUS_OBS_MERGE_FLAG} + // ${METPLUS_OBS_FILE_TYPE} + multivar_name = "Precip"; + multivar_level = "LO"; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Handle missing data +// +// mask_missing_flag = +// ${METPLUS_MASK_MISSING_FLAG} + +// +// Match objects between the forecast and observation fields +// +//match_flag = +//${METPLUS_MATCH_FLAG} +match_flag = MERGE_BOTH; +// +// Maximum centroid distance for objects to be compared +// +//max_centroid_dist = +//${METPLUS_MAX_CENTROID_DIST} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +//mask = { +//${METPLUS_MASK_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fuzzy engine weights +// +//weight = { +//${METPLUS_WEIGHT_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fuzzy engine interest functions +// +interest_function = { + + //${METPLUS_INTEREST_FUNCTION_CENTROID_DIST} + + //${METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST} + + //${METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST} + + angle_diff = ( + ( 0.0, 1.0 ) + ( 30.0, 1.0 ) + ( 90.0, 0.0 ) + ); + + aspect_diff = ( + ( 0.00, 1.0 ) + ( 0.10, 1.0 ) + ( 0.75, 0.0 ) + ); + + corner = 0.8; + ratio_if = ( + ( 0.0, 0.0 ) + ( corner, 1.0 ) + ( 1.0, 1.0 ) + ); + + area_ratio = ratio_if; + + int_area_ratio = ( + ( 0.00, 0.00 ) + ( 0.10, 0.50 ) + ( 0.25, 1.00 ) + ( 1.00, 1.00 ) + ); + + curvature_ratio = ratio_if; + + complexity_ratio = ratio_if; + + inten_perc_ratio = ratio_if; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Total interest threshold for determining matches +// +//total_interest_thresh = +//${METPLUS_TOTAL_INTEREST_THRESH} + +// +// Interest threshold for printing output pair information +// +print_interest_thresh = 0.0; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Plotting information +// +met_data_dir = "MET_BASE"; + +fcst_raw_plot = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; +} + +obs_raw_plot = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; +} + +object_plot = { + color_table = "MET_BASE/colortables/mode_obj.ctable"; +} + +// +// Boolean for plotting on the region of valid data within the domain +// +plot_valid_flag = FALSE; + +// +// Plot polyline edges using great circle arcs instead of straight lines +// +plot_gcarc_flag = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// NetCDF matched pairs, PostScript, and contingency table output files +// +//ps_plot_flag = +//${METPLUS_PS_PLOT_FLAG} + +//nc_pairs_flag = { +//${METPLUS_NC_PAIRS_FLAG_DICT} + +//ct_stats_flag = +//${METPLUS_CT_STATS_FLAG} + + +//////////////////////////////////////////////////////////////////////////////// + +shift_right = 0; // grid squares + +//////////////////////////////////////////////////////////////////////////////// + +//${METPLUS_OUTPUT_PREFIX} +//version = "V10.0"; + +//tmp_dir = "${MET_TMP_DIR}"; +tmp_dir = "/tmp"; + +//////////////////////////////////////////////////////////////////////////////// + +//${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/internal/test_unit/config/MODEConfig_multivar_super b/internal/test_unit/config/MODEConfig_multivar_super new file mode 100644 index 0000000000..e035f6ba63 --- /dev/null +++ b/internal/test_unit/config/MODEConfig_multivar_super @@ -0,0 +1,271 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// MODE configuration file. +// +// For additional information, see the MET_BASE/config/README file. +// +//////////////////////////////////////////////////////////////////////////////// + +// +// Output model name to be written +// +// model = +model = "HRRR"; + +// +// Output description to be written +// +// desc = + +// +// Output observation type to be written +// +// obtype = +obtype = "ANALYSIS"; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification grid +// +regrid = {to_grid = FCST;method = NEAREST;width = 1;vld_thresh = 0.5;} + +//////////////////////////////////////////////////////////////////////////////// + +// +`// Approximate grid resolution (km) +// +// grid_res = +// + +//////////////////////////////////////////////////////////////////////////////// + +// +// Run all permutations of radius and threshold +// +// quilt = +// ${METPLUS_QUILT} + +// +// MODE Multivar boolean combination logic +// +//multivar_logic = +//${METPLUS_MULTIVAR_LOGIC} +multivar_logic = "#1 && #2 && #3"; + +multivar_intensity_compare_fcst = [ ]; +multivar_intensity_compare_obs = [ ]; + +// +// Forecast and observation fields to be verified +// +fcst = { + //${METPLUS_FCST_FIELD} + field = [{ name="CSNOW"; level="L0"; conv_radius = 0; conv_thresh = ==1; file_type = GRIB2; merge_flag = NONE; }, + { name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; }, + { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; } ]; + + //${METPLUS_FCST_CENSOR_THRESH} + //${METPLUS_FCST_CENSOR_VAL} + //${METPLUS_FCST_CONV_RADIUS} + //${METPLUS_FCST_CONV_THRESH} + //${METPLUS_FCST_VLD_THRESH} + //${METPLUS_FCST_FILTER_ATTR_NAME} + filter_attr_name = ["AREA"]; + + //${METPLUS_FCST_FILTER_ATTR_THRESH} + filter_attr_thresh = [>=25]; + + //${METPLUS_FCST_MERGE_THRESH} + //${METPLUS_FCST_MERGE_FLAG} + //${METPLUS_FCST_FILE_TYPE} + //file_type = GRIB2; + multivar_name = "Snow"; + multivar_level = "LO"; +} + +obs = { + //${METPLUS_OBS_FIELD} + field = [{ name="PrecipFlag"; level="L0"; conv_radius = 0; conv_thresh = ==3; file_type = GRIB2; merge_flag = NONE; }, + { name="VIS"; level="L0"; conv_radius = 5; conv_thresh = <=804.672; merge_thresh = <=1207.008; merge_flag = THRESH; }, + { name="WIND"; level="Z10"; conv_radius = 5; conv_thresh = >=8.9408; merge_thresh = >=6.7056; merge_flag = THRESH; } ]; + + //${METPLUS_OBS_CENSOR_THRESH} + //${METPLUS_OBS_CENSOR_VAL} + //${METPLUS_OBS_CONV_RADIUS} + //${METPLUS_OBS_CONV_THRESH} + //${METPLUS_OBS_VLD_THRESH} + + //${METPLUS_OBS_FILTER_ATTR_NAME} + filter_attr_name = ["AREA"]; + + //${METPLUS_OBS_FILTER_ATTR_THRESH} + filter_attr_thresh = [>=25]; + + // ${METPLUS_OBS_MERGE_THRESH} + // ${METPLUS_OBS_MERGE_FLAG} + // ${METPLUS_OBS_FILE_TYPE} + multivar_name = "Precip"; + multivar_level = "LO"; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Handle missing data +// +// mask_missing_flag = +// ${METPLUS_MASK_MISSING_FLAG} + +// +// Match objects between the forecast and observation fields +// +//match_flag = +//${METPLUS_MATCH_FLAG} +match_flag = MERGE_BOTH; +// +// Maximum centroid distance for objects to be compared +// +//max_centroid_dist = +//${METPLUS_MAX_CENTROID_DIST} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Verification masking regions +// +//mask = { +//${METPLUS_MASK_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fuzzy engine weights +// +//weight = { +//${METPLUS_WEIGHT_DICT} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Fuzzy engine interest functions +// +interest_function = { + + //${METPLUS_INTEREST_FUNCTION_CENTROID_DIST} + + //${METPLUS_INTEREST_FUNCTION_BOUNDARY_DIST} + + //${METPLUS_INTEREST_FUNCTION_CONVEX_HULL_DIST} + + angle_diff = ( + ( 0.0, 1.0 ) + ( 30.0, 1.0 ) + ( 90.0, 0.0 ) + ); + + aspect_diff = ( + ( 0.00, 1.0 ) + ( 0.10, 1.0 ) + ( 0.75, 0.0 ) + ); + + corner = 0.8; + ratio_if = ( + ( 0.0, 0.0 ) + ( corner, 1.0 ) + ( 1.0, 1.0 ) + ); + + area_ratio = ratio_if; + + int_area_ratio = ( + ( 0.00, 0.00 ) + ( 0.10, 0.50 ) + ( 0.25, 1.00 ) + ( 1.00, 1.00 ) + ); + + curvature_ratio = ratio_if; + + complexity_ratio = ratio_if; + + inten_perc_ratio = ratio_if; +} + +//////////////////////////////////////////////////////////////////////////////// + +// +// Total interest threshold for determining matches +// +//total_interest_thresh = +//${METPLUS_TOTAL_INTEREST_THRESH} + +// +// Interest threshold for printing output pair information +// +print_interest_thresh = 0.0; + +//////////////////////////////////////////////////////////////////////////////// + +// +// Plotting information +// +met_data_dir = "MET_BASE"; + +fcst_raw_plot = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; +} + +obs_raw_plot = { + color_table = "MET_BASE/colortables/met_default.ctable"; + plot_min = 0.0; + plot_max = 0.0; +} + +object_plot = { + color_table = "MET_BASE/colortables/mode_obj.ctable"; +} + +// +// Boolean for plotting on the region of valid data within the domain +// +plot_valid_flag = FALSE; + +// +// Plot polyline edges using great circle arcs instead of straight lines +// +plot_gcarc_flag = FALSE; + +//////////////////////////////////////////////////////////////////////////////// + +// +// NetCDF matched pairs, PostScript, and contingency table output files +// +//ps_plot_flag = +//${METPLUS_PS_PLOT_FLAG} + +//nc_pairs_flag = { +//${METPLUS_NC_PAIRS_FLAG_DICT} + +//ct_stats_flag = +//${METPLUS_CT_STATS_FLAG} + + +//////////////////////////////////////////////////////////////////////////////// + +shift_right = 0; // grid squares + +//////////////////////////////////////////////////////////////////////////////// + +//${METPLUS_OUTPUT_PREFIX} +//version = "V10.0"; + +//tmp_dir = "${MET_TMP_DIR}"; +tmp_dir = "/tmp"; + +//////////////////////////////////////////////////////////////////////////////// + +//${METPLUS_MET_CONFIG_OVERRIDES} diff --git a/internal/test_unit/xml/unit_mode_multivar.xml b/internal/test_unit/xml/unit_mode_multivar.xml index 4aad5dc7d6..aea7f7f2a0 100644 --- a/internal/test_unit/xml/unit_mode_multivar.xml +++ b/internal/test_unit/xml/unit_mode_multivar.xml @@ -16,7 +16,90 @@ &TEST_DIR; true - + + echo "&DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2 \ + &DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2 \ + &DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2" \ + > &OUTPUT_DIR;/mode_multivar/input_fcst_file_list; \ + echo "&DATA_DIR_OBS;/mode_multivar/mrms/20210201/PrecipFlag_00.00_20210201-210000.sub.grib2 \ + &DATA_DIR_OBS;/mode_multivar/hrrr/anl/20210201/hrrr.t21z.wrfprsf00.sub.grib2 \ + &DATA_DIR_OBS;/mode_multivar/hrrr/anl/20210201/hrrr.t21z.wrfprsf00.sub.grib2" \ + > &OUTPUT_DIR;/mode_multivar/input_obs_file_list; \ + &MET_BIN;/mode + \ + &OUTPUT_DIR;/mode_multivar/input_fcst_file_list \ + &OUTPUT_DIR;/mode_multivar/input_obs_file_list \ + &CONFIG_DIR;/MODEConfig_multivar \ + -outdir &OUTPUT_DIR;/mode_multivar \ + -v 2 + + + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A_obj.nc + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A_obj.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A_cts.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A.ps + &OUTPUT_DIR;/mode_multivar/mode_Fcst_WIND_Z10_Obs_WIND_Z10_210000L_20210201_210000V_000000A_obj.nc + &OUTPUT_DIR;/mode_multivar/mode_Fcst_WIND_Z10_Obs_WIND_Z10_210000L_20210201_210000V_000000A_obj.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_WIND_Z10_Obs_WIND_Z10_210000L_20210201_210000V_000000A_cts.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_WIND_Z10_Obs_WIND_Z10_210000L_20210201_210000V_000000A.ps + + + + + + echo "&DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2 \ + &DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2 \ + &DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2" \ + > &OUTPUT_DIR;/mode_multivar/input_fcst_file_list; \ + echo "&DATA_DIR_OBS;/mode_multivar/hrrr/anl/20210201/hrrr.t21z.wrfprsf00.sub.grib2 \ + &DATA_DIR_OBS;/mode_multivar/hrrr/anl/20210201/hrrr.t21z.wrfprsf00.sub.grib2" \ + > &OUTPUT_DIR;/mode_multivar/input_obs_file_list; \ + &MET_BIN;/mode + \ + &OUTPUT_DIR;/mode_multivar/input_fcst_file_list \ + &OUTPUT_DIR;/mode_multivar/input_obs_file_list \ + &CONFIG_DIR;/MODEConfig_multivar_3_2 \ + -outdir &OUTPUT_DIR;/mode_multivar \ + -v 2 + + + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A_obj.nc + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A_obj.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A_cts.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_VIS_L0_210000L_20210201_210000V_000000A.ps + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_WIND_Z10_210000L_20210201_210000V_000000A_obj.nc + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_WIND_Z10_210000L_20210201_210000V_000000A_obj.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_WIND_Z10_210000L_20210201_210000V_000000A_cts.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_VIS_L0_Obs_WIND_Z10_210000L_20210201_210000V_000000A.ps + + + + + echo "&DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2 \ + &DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2 \ + &DATA_DIR_MODEL;/mode_multivar/hrrr/2021020100/hrrr.t00z.wrfprsf21.sub.grib2" \ + > &OUTPUT_DIR;/mode_multivar/input_fcst_file_list; \ + echo "&DATA_DIR_OBS;/mode_multivar/mrms/20210201/PrecipFlag_00.00_20210201-210000.sub.grib2 \ + &DATA_DIR_OBS;/mode_multivar/hrrr/anl/20210201/hrrr.t21z.wrfprsf00.sub.grib2 \ + &DATA_DIR_OBS;/mode_multivar/hrrr/anl/20210201/hrrr.t21z.wrfprsf00.sub.grib2" \ + > &OUTPUT_DIR;/mode_multivar/input_obs_file_list; \ + &MET_BIN;/mode + \ + &OUTPUT_DIR;/mode_multivar/input_fcst_file_list \ + &OUTPUT_DIR;/mode_multivar/input_obs_file_list \ + &CONFIG_DIR;/MODEConfig_multivar_super \ + -outdir &OUTPUT_DIR;/mode_multivar \ + -v 2 + + + &OUTPUT_DIR;/mode_multivar/mode_Fcst_Snow_LO_Obs_Precip_LO_210000L_20210201_210000V_000000A_obj.nc + &OUTPUT_DIR;/mode_multivar/mode_Fcst_Snow_LO_Obs_Precip_LO_210000L_20210201_210000V_000000A_obj.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_Snow_LO_Obs_Precip_LO_210000L_20210201_210000V_000000A_cts.txt + &OUTPUT_DIR;/mode_multivar/mode_Fcst_Snow_LO_Obs_Precip_LO_210000L_20210201_210000V_000000A.ps + + + + echo "&DATA_DIR_MODEL;/mode_multivar/alpha_fcst.nc \ &DATA_DIR_MODEL;/mode_multivar/beta_fcst.nc \ &DATA_DIR_MODEL;/mode_multivar/gamma_fcst.nc" \ diff --git a/src/basic/vx_config/config_file.cc b/src/basic/vx_config/config_file.cc index 67c8af8959..31dd4fa60c 100644 --- a/src/basic/vx_config/config_file.cc +++ b/src/basic/vx_config/config_file.cc @@ -115,6 +115,20 @@ assign(c); } +//////////////////////////////////////////////////////////////////////// + +MetConfig & MetConfig::operator=(const MetConfig &s) +{ + if(this == &s) return(*this); + + init_from_scratch(); + + assign(s); + + return(*this); + +} + //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_shapedata/Makefile.am b/src/libcode/vx_shapedata/Makefile.am index f54e937535..b5fe46ed68 100644 --- a/src/libcode/vx_shapedata/Makefile.am +++ b/src/libcode/vx_shapedata/Makefile.am @@ -21,6 +21,7 @@ libvx_shapedata_a_SOURCES = \ mode_columns.h \ mode_field_info.cc mode_field_info.h \ mode_conf_info.cc mode_conf_info.h \ + mode_input_data.h \ engine.cc engine.h \ ihull.cc ihull.h \ vx_shapedata.h diff --git a/src/libcode/vx_shapedata/engine.cc b/src/libcode/vx_shapedata/engine.cc index e81c2d21bc..dd041dc9c1 100644 --- a/src/libcode/vx_shapedata/engine.cc +++ b/src/libcode/vx_shapedata/engine.cc @@ -144,6 +144,7 @@ void ModeFuzzyEngine::init_from_scratch() { // // Reset all fcst and obs processing flags to initial state // + need_fcst_conv = true; need_fcst_thresh = true; need_fcst_filter = true; @@ -1724,8 +1725,8 @@ void ModeFuzzyEngine::do_fcst_merge_engine(const char *default_config, fcst_engine->ctable = ctable; if(default_config && merge_config) { fcst_engine->conf_info.read_config(default_config, merge_config); - fcst_engine->conf_info.process_config(conf_info.Fcst->var_info->file_type(), - conf_info.Obs->var_info->file_type()); + fcst_engine->conf_info.process_config_traditional(conf_info.Fcst->var_info->file_type(), + conf_info.Obs->var_info->file_type()); path = replace_path(fcst_engine->conf_info.object_pi.color_table.c_str()); fcst_engine->ctable.read(path.c_str()); } @@ -1891,8 +1892,8 @@ void ModeFuzzyEngine::do_obs_merge_engine(const char *default_config, obs_engine->ctable = ctable; if(default_config && merge_config) { obs_engine->conf_info.read_config(default_config, merge_config); - obs_engine->conf_info.process_config(conf_info.Fcst->var_info->file_type(), - conf_info.Obs->var_info->file_type()); + obs_engine->conf_info.process_config_traditional(conf_info.Fcst->var_info->file_type(), + conf_info.Obs->var_info->file_type()); path = replace_path(obs_engine->conf_info.object_pi.color_table.c_str()); obs_engine->ctable.read(path.c_str()); } diff --git a/src/libcode/vx_shapedata/mode_conf_info.cc b/src/libcode/vx_shapedata/mode_conf_info.cc index a2ebee3c5b..615e3f90eb 100644 --- a/src/libcode/vx_shapedata/mode_conf_info.cc +++ b/src/libcode/vx_shapedata/mode_conf_info.cc @@ -17,6 +17,7 @@ using namespace std; #include #include #include +#include #include "mode_conf_info.h" #include "configobjecttype_to_string.h" @@ -51,6 +52,16 @@ ModeConfInfo::~ModeConfInfo() clear(); } +ModeConfInfo & ModeConfInfo::operator=(const ModeConfInfo &s) +{ + if(this == &s) return(*this); + + clear(); + assign(s); + + return(*this); +} + //////////////////////////////////////////////////////////////////////// void ModeConfInfo::init_from_scratch() @@ -77,6 +88,99 @@ void ModeConfInfo::init_from_scratch() //////////////////////////////////////////////////////////////////////// +void ModeConfInfo::assign( const ModeConfInfo &m) +{ + Field_Index_f = m.Field_Index_f; + Field_Index_o = m.Field_Index_o; + N_fields_f = m.N_fields_f; + N_fields_o = m.N_fields_o; + fcst_multivar_logic = m.fcst_multivar_logic; + obs_multivar_logic = m.obs_multivar_logic; + fcst_multivar_compare_index = m.fcst_multivar_compare_index; + obs_multivar_compare_index = m.obs_multivar_compare_index; + fcst_multivar_name = m.fcst_multivar_name; + fcst_multivar_level = m.fcst_multivar_level; + obs_multivar_name = m.obs_multivar_name; + obs_multivar_level = m.obs_multivar_level; + data_type = m.data_type; + conf = m.conf; + centroid_dist_wt = m.centroid_dist_wt; + boundary_dist_wt = m.boundary_dist_wt; + convex_hull_dist_wt = m.convex_hull_dist_wt; + angle_diff_wt = m.angle_diff_wt; + aspect_diff_wt = m.aspect_diff_wt; + area_ratio_wt = m.area_ratio_wt; + int_area_ratio_wt = m.int_area_ratio_wt; + curvature_ratio_wt = m.curvature_ratio_wt; + complexity_ratio_wt = m.complexity_ratio_wt; + inten_perc_ratio_wt = m.inten_perc_ratio_wt; + + fcst_array = 0; + Fcst = 0; + if (N_fields_f > 0) { + fcst_array = new Mode_Field_Info[N_fields_f]; + for (int i=0; i 0) { + obs_array = new Mode_Field_Info[N_fields_o]; + for (int i=0; ilookup_int(conf_key_shift_right); - read_fields (fcst_array, fcst_dict, ftype, 'F'); // the order is important here - read_fields ( obs_array, obs_dict, otype, 'O'); // the order is important here - Fcst = fcst_array; // + 0 - Obs = obs_array; // + 0 + if (field_index == 0) { + read_fields_0 (fcst_array, fcst_dict, ftype, 'F'); + read_fields_1 (fcst_array, fcst_dict, ftype, 'F', 0); + read_fields_0 (obs_array, obs_dict, otype, 'O'); + read_fields_1 (obs_array, obs_dict, otype, 'O', 0); + fcst_array->raw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_fcst_raw_plot)); + obs_array->raw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_obs_raw_plot)); + + } else { + read_fields_1 (fcst_array, fcst_dict, ftype, 'F', field_index); + read_fields_1 (obs_array, obs_dict, otype, 'O', field_index); + } + + Fcst = fcst_array + field_index; + Obs = obs_array + field_index; // Dump the contents of the VarInfo objects if(mlog.verbosity_level() >= 5) { - for (j=0; jdump(cout); - } - for (j=0; jdump(cout); - } // for j - } - for (j=0; jdump(cout); + mlog << Debug(5) + << "Parsed observation field:\n"; + obs_array[field_index].var_info->dump(cout); } - for (j=0; j " - << "fcst and obs convolution radius arrays need to be the same size for traditional mode\n\n"; - exit ( 1 ); - } - if ( fcst_array[j].conv_thresh_array.n_elements() != obs_array[j].conv_thresh_array.n_elements() ) { + if ( fcst_array[field_index].conv_radius_array.n_elements() != + obs_array[field_index].conv_radius_array.n_elements() ) { + mlog << Error << "\nModeConfInfo::process_config_both() -> " + << "fcst and obs convolution radius arrays need to be the same size for traditional mode\n\n"; + exit ( 1 ); + } + if ( fcst_array[field_index].conv_thresh_array.n_elements() != + obs_array[field_index].conv_thresh_array.n_elements() ) { - mlog << Error << "\nModeConfInfo::process_config_both() -> " - << "fcst and obs convolution threshold arrays need to be the same size\n\n"; - exit ( 1 ); - } + mlog << Error << "\nModeConfInfo::process_config_both() -> " + << "fcst and obs convolution threshold arrays need to be the same size\n\n"; + exit ( 1 ); } } - // Conf: fcst_raw_plot - - fcst_array->raw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_fcst_raw_plot)); - - // Conf: obs_raw_plot - - obs_array->raw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_obs_raw_plot)); - - return; + return; } //////////////////////////////////////////////////////////////////////// -void ModeConfInfo::process_config_fcst(GrdFileType ftype) +void ModeConfInfo::process_config_fcst(GrdFileType ftype, int field_index) { int j, k, n; @@ -501,25 +613,26 @@ void ModeConfInfo::process_config_fcst(GrdFileType ftype) fcst_dict = conf.lookup_dictionary(conf_key_fcst); shift_right = fcst_dict->lookup_int(conf_key_shift_right); - read_fields (fcst_array, fcst_dict, ftype, 'F'); // the order is important here - Fcst = fcst_array; // + 0 + if (field_index == 0) { + read_fields_0 (fcst_array, fcst_dict, ftype, 'F'); + read_fields_1 (fcst_array, fcst_dict, ftype, 'F', 0); + fcst_array->raw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_fcst_raw_plot)); + } else { + read_fields_1 (fcst_array, fcst_dict, ftype, 'F', field_index); + } + + Fcst = fcst_array + field_index; - // Dump the contents of the VarInfo objects + // Dump the contents of the VarInfo object if(mlog.verbosity_level() >= 5) { - for (j=0; jdump(cout); - } // for j + mlog << Debug(5) + << "Parsed forecast field:\n"; + fcst_array[field_index].var_info->dump(cout); } - for (j=0; jraw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_fcst_raw_plot)); + evaluate_fcst_settings(field_index); return; @@ -528,7 +641,7 @@ void ModeConfInfo::process_config_fcst(GrdFileType ftype) //////////////////////////////////////////////////////////////////////// -void ModeConfInfo::process_config_obs(GrdFileType otype) +void ModeConfInfo::process_config_obs(GrdFileType otype, int field_index) { int j, k, n; @@ -537,25 +650,25 @@ void ModeConfInfo::process_config_obs(GrdFileType otype) obs_dict = conf.lookup_dictionary(conf_key_obs); shift_right = obs_dict->lookup_int(conf_key_shift_right); - read_fields ( obs_array, obs_dict, otype, 'O'); // the order is important here - Obs = obs_array; // + 0 + if (field_index == 0) { + read_fields_0 (obs_array, obs_dict, otype, 'O'); + read_fields_1 (obs_array, obs_dict, otype, 'O', 0); + obs_array->raw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_obs_raw_plot)); + } else { + read_fields_1 (obs_array, obs_dict, otype, 'O', field_index); + } + Obs = obs_array + field_index; - // Dump the contents of the VarInfo objects + // Dump the contents of the VarInfo object if(mlog.verbosity_level() >= 5) { - for (j=0; jdump(cout); - } // for j + mlog << Debug(5) + << "Parsed observation field:\n"; + obs_array[field_index].var_info->dump(cout); } - for (j=0; jraw_pi = parse_conf_plot_info(conf.lookup_dictionary(conf_key_obs_raw_plot)); + evaluate_obs_settings(field_index); return; @@ -563,6 +676,127 @@ void ModeConfInfo::process_config_obs(GrdFileType otype) //////////////////////////////////////////////////////////////////////// +void ModeConfInfo::config_set_all_percentile_thresholds(const std::vector &fdata, + const std::vector &odata) +{ + + // for each forecast input, it's either not a percentile threshold, is a simple (single input) percentile, + // a frequency bias percentile threshold, or a climatology percentile (we don't have climatology). + // as a complication we have both conv_thresh and merge_thresh, so what if they are different? + // right now I simply go with freaquency bias as highest priority + vector fcst_freq, obs_freq; + for (int j=0; j= N_fields_o) { + mlog << Error << "\nModeConfInfo::config_set_all_percentile_thresholds\n" + << " Frequency bias Thresholding on fcst index " << j+1 + << " out of range of obs " << N_fields_o + 1 << "\n\n"; + exit ( 1 ); + } + // save for later + break; + default: + break; + } + + if (need_perc) { + data_type = ModeDataType_MvMode_Fcst; + set_field_index(j); + set_perc_thresh(fdata[j]._dataPlane); + } + if (need_freq) { + fcst_freq.push_back(j); + } + } + + for (int j=0; j= N_fields_f) { + mlog << Error << "\nModeConfInfo::config_set_all_percentile_thresholds\n" + << " Frequency bias Thresholding on obs index " << j+1 + << " out of range of fcst " << N_fields_f + 1 << "\n\n"; + exit ( 1 ); + } + // save for later + break; + default: + break; + } + + if (need_perc) { + data_type = ModeDataType_MvMode_Obs; + set_field_index(j); + set_perc_thresh(odata[j]._dataPlane); + } + if (need_freq) { + obs_freq.push_back(j); + } + } + + // make sure no overlap in fcst/obs frequencies + for (size_t i=0; iis_array() ) { exit ( 1 ); } + } - info_array[j].set(true, j, e->dict_value(), &conf, type, _fo, false); + return; - if ( j == 0 ) { +} // if is array - for (k=1; klookup(conf_key_field); + if ( !ee ) { + + mlog << "\nModeConfInfo::read_fields () -> \"field\" entry not found in dictionary!\n\n"; + + exit ( 1 ); + +} + +const Dictionary * field = ee->dict(); + +const int N = ( (field->is_array()) ? (field->n_entries()) : 1 ); + +if ( field->is_array() ) { + + const DictionaryEntry * e = 0; + const Dictionary & D = *field; + + e = D[field_index]; + if ( (e->type() != DictionaryType) && (e->type() != ArrayType) ) { + mlog << Error + << "\nModeConfInfo::read_fields() -> field entry # " << field_index+1 << " is not a dictionary!\n\n"; + exit ( 1 ); + + } + + info_array[field_index].set(true, field_index, e->dict_value(), &conf, type, _fo, false); return; } // if is array - // // nope, traditional mode // -N_fields_f = N; -N_fields_o = N; + if (field_index != 0) { + mlog << Error + << "\nModeConfInfo::read_fields() -> traditional mode but looking for a field array index " << field_index + << "\n\n"; + exit ( 1 ); + } + + info_array[0].set(false, 0, dict, &conf, type, _fo); // @@ -1287,6 +1569,46 @@ void ModeConfInfo::set_data_type(ModeDataType type) //////////////////////////////////////////////////////////////////////// +GrdFileType ModeConfInfo::file_type_for_field(bool isFcst, int field_index) +{ + // look at the dictionary for the obs or forecast at index, with + // parents + + Dictionary * dict = (Dictionary *) 0; + if (isFcst) { + dict = conf.lookup_dictionary(conf_key_fcst); + } else { + dict = conf.lookup_dictionary(conf_key_obs); + } + + const DictionaryEntry *e = dict->lookup(conf_key_field); + if ( !e ) { + mlog << "\nModeConfInfo::read_fields () -> \"field\" entry not found in dictionary!\n\n"; + exit ( 1 ); + } + + const Dictionary * field = e->dict(); + const int N = ( (field->is_array()) ? (field->n_entries()) : 1 ); + if (field_index < 0 || field_index >= N) { + mlog << Error + << "\nModeConfInfo::file_type_for_field() -> " + << "index " << field_index+1 << " out of range 0 to " << N + << "\n\n"; + exit (1); + } + e = (*field)[field_index]; + if ( (e->type() != DictionaryType) && (e->type() != ArrayType) ) { + mlog << Error + << "\nModeConfInfo::file_type_for_field() -> " + << "field entry # " << field_index+1 << " is not a dictionary!\n\n"; + exit ( 1 ); + } + Dictionary *dictf = e->dict_value(); + return parse_conf_file_type(dictf); +} + +//////////////////////////////////////////////////////////////////////// + int ModeConfInfo::n_runs() const @@ -1406,7 +1728,12 @@ void ModeConfInfo::check_multivar_not_implemented() << "\nModeConfInfo::check_multivar_not_implemented():\n" << " merge_flag ENGINE or BOTH not implemented for multivariate mode\n\n"; status = true; - break; + } + if (fcst_array[i].conv_thresh_array.n() > 1 || fcst_array[i].merge_thresh_array.n() > 1) { + mlog << Error + << "\nModeConfInfo::check_multivar_not_implemented():\n" + << " more than one conv_thresh or merge_thresh per input is not allowed in multivariate mode\n\n"; + status = true; } } } @@ -1419,6 +1746,12 @@ void ModeConfInfo::check_multivar_not_implemented() status = true; break; } + if (obs_array[i].conv_thresh_array.n() > 1 || obs_array[i].merge_thresh_array.n() > 1) { + mlog << Error + << "\nModeConfInfo::check_multivar_not_implemented():\n" + << " more than one conv_thresh or merge_thresh per input is not allowed in multivariate mode\n\n"; + status = true; + } } } @@ -1433,52 +1766,43 @@ void ModeConfInfo::check_multivar_not_implemented() //////////////////////////////////////////////////////////////////////// -void ModeConfInfo::check_multivar_perc_thresh(bool isSimple, bool isSimpleMerge) const +PercThreshType ModeConfInfo::perctype(const Mode_Field_Info &f) const { - if (isSimple) { - if (data_type == ModeDataType_MvMode_Fcst) { - for (int j=0; j< Fcst->conv_thresh_array.n(); ++j) { - if (Fcst->conv_thresh_array[j].get_ptype() == perc_thresh_sample_obs) { - mlog << Warning - << "\nModeConfInfo::check_multivar_perc_thresh:\n" - << " Thresholding with 'SOP' in a forecast input not well defined for multivariate mode simple object creation\n" - << " 'SFP' will be used as a replacement\n\n"; - } - } - } else if (data_type == ModeDataType_MvMode_Obs) { - for (int j=0; j< Obs->conv_thresh_array.n(); ++j) { - if (Obs->conv_thresh_array[j].get_ptype() == perc_thresh_sample_fcst) { - mlog << Warning - << "\nModeConfInfo::check_multivar_perc_thresh:\n" - << " Thresholding with 'SFP' in an obs input not well defined for multivariate mode simple object creation\n" - << " 'SOP' will be used as a replacement\n\n"; - } - } - } + PercThreshType pm=no_perc_thresh_type;; + PercThreshType pc=no_perc_thresh_type;; + if (f.merge_thresh_array.n() > 0) { + pm = f.merge_thresh_array[0].get_ptype(); } - - if (isSimpleMerge) { - if (data_type == ModeDataType_MvMode_Fcst) { - for (int j=0; j< Fcst->merge_thresh_array.n(); ++j) { - if (Fcst->merge_thresh_array[j].get_ptype() == perc_thresh_sample_obs) { - mlog << Warning - << "\nModeConfInfo::check_multivar_perc_thresh:\n" - << " Thresholding with 'SOP' in a forecast input not well defined for multivariate mode simple object creation\n" - << " 'SFP' will be used as a replacement\n\n"; - } - } - } else if (data_type == ModeDataType_MvMode_Obs) { - for (int j=0; j< Obs->merge_thresh_array.n(); ++j) { - if (Obs->merge_thresh_array[j].get_ptype() == perc_thresh_sample_fcst) { - mlog << Warning - << "\nModeConfInfo::check_multivar_perc_thresh():\n" - << " Thresholding with 'SFP' in an obs input not well defined for multivariate mode simple object creation\n" - << " 'SOP' will be used as a replacement\n\n"; - } - } - } + if (f.conv_thresh_array.n() > 0) { + pc = f.conv_thresh_array[0].get_ptype(); } -} + if (pm == perc_thresh_sample_climo || pc == perc_thresh_sample_climo) { + mlog << Error << "\nModeConfInfo::perctype()\n" + << " Thresholding with 'SCP' in an input not implemented for multivariate mode\n\n"; + exit ( 1 ); + } + if (pm == perc_thresh_climo_dist || pc == perc_thresh_climo_dist) { + mlog << Error << "\nModeConfInfo::perctype()\n" + << " Thresholding with 'CDP' in an input not implemented for multivariate mode\n\n"; + exit ( 1 ); + } + if (pm == perc_thresh_freq_bias || + pc == perc_thresh_freq_bias) { + return perc_thresh_freq_bias; + } + // put in tests if they are not the same and + // both are sample settings, so the right one + // gets returned (pass in fcst/obs boolean) + if (pm == perc_thresh_sample_obs || + pc == perc_thresh_sample_obs) { + return perc_thresh_sample_obs; + } + if (pm == perc_thresh_sample_fcst || + pc == perc_thresh_sample_fcst) { + return perc_thresh_sample_fcst; + } + return no_perc_thresh_type; +} //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_shapedata/mode_conf_info.h b/src/libcode/vx_shapedata/mode_conf_info.h index 7fab368dc4..fcaca3e5a0 100644 --- a/src/libcode/vx_shapedata/mode_conf_info.h +++ b/src/libcode/vx_shapedata/mode_conf_info.h @@ -25,6 +25,7 @@ #include "mode_field_info.h" #include "mode_data_type.h" +#include "mode_input_data.h" //////////////////////////////////////////////////////////////////////// @@ -74,12 +75,17 @@ class ModeConfInfo { int N_fields_f; // for traditional mode expect N_fields_f/_o to be the same int N_fields_o; + void assign(const ModeConfInfo &); + public: ModeConfInfo(); ~ModeConfInfo(); + // attempt to implement this + ModeConfInfo & operator=(const ModeConfInfo &); + void clear(); // sets both obs and fcst field indices if traditional mode, or mvmode 'both' @@ -106,7 +112,15 @@ class ModeConfInfo { void check_multivar_not_implemented(); - void check_multivar_perc_thresh(bool isSimple, bool isSimpleMerge) const; + // check all inputs for climatology percentiles, which are not implemented + // warn about forecasts and obs set backwards + // void check_multivar_perc_thresh(const Mode_Field_Info &f, bool isFcst) const; + + + // // return index to other input if the particular input is a frequency bias percentile + // // return -1 if input is not a fequency bias percentile + // // if input is forecast, 'other' is obs, and vice versa + // int frequency_bias_other_index(bool input_is_fcst, int input_index) const; ///////////////////////////////////////////////////////////////////// @@ -122,7 +136,7 @@ class ModeConfInfo { // and to distinguish traditional mode from mvmode ModeDataType data_type; - + ///////////////////////////////////////////////////////////////////// @@ -135,9 +149,21 @@ class ModeConfInfo { void read_config (const char * default_filename, const char * user_filename); - void process_config (GrdFileType ftype, GrdFileType otype, ModeDataType dt=ModeDataType_Traditional); + void process_config_traditional(GrdFileType ftype, GrdFileType otype); - void read_fields (Mode_Field_Info * &, Dictionary * dict, GrdFileType, char _fo); + void process_config_except_fields(); + + void process_config_field (GrdFileType ftype, GrdFileType otype, ModeDataType dt, int field_index); + + void config_set_all_percentile_thresholds(const std::vector &fdata, + const std::vector &odata); + PercThreshType perctype(const Mode_Field_Info &f) const; + + // deal with zeroth field + void read_fields_0 (Mode_Field_Info * &, Dictionary * dict, GrdFileType, char _fo); + + // deal with non-zeroth field + void read_fields_1 (Mode_Field_Info * &, Dictionary * dict, GrdFileType, char _fo, int field_index); PiecewiseLinear * parse_interest_function(Dictionary * dict, const char * conf_key_if); @@ -269,12 +295,14 @@ class ModeConfInfo { void set_data_type(ModeDataType type); + GrdFileType file_type_for_field(bool isFcst, int field_index); + private: // some private methods - void process_config_both(GrdFileType ftype, GrdFileType otype); - void process_config_fcst(GrdFileType ftype); - void process_config_obs(GrdFileType otype); + void process_config_both(GrdFileType ftype, GrdFileType otype, int field_index=0); + void process_config_fcst(GrdFileType ftype, int field_index=0); + void process_config_obs(GrdFileType otype, int field_index=0); void evaluate_fcst_settings(int); void evaluate_obs_settings(int); diff --git a/src/libcode/vx_shapedata/mode_field_info.cc b/src/libcode/vx_shapedata/mode_field_info.cc index 8dd024e803..9569dac765 100644 --- a/src/libcode/vx_shapedata/mode_field_info.cc +++ b/src/libcode/vx_shapedata/mode_field_info.cc @@ -85,6 +85,38 @@ return ( * this ); } +//////////////////////////////////////////////////////////////////////// + +void Mode_Field_Info::clone(const Mode_Field_Info & i) +{ + dict = i.dict; + conf = i.conf; + gft = i.gft; + FO = i.FO; + Multivar = i.Multivar; + index = i.index; + conv_radius = i.conv_radius; + vld_thresh = i.vld_thresh; + VarInfoFactory info_factory; + var_info = info_factory.new_var_info(gft); + if ( Multivar ) { + var_info->set_dict(*dict); + } else { + var_info->set_dict(*(dict->lookup_dictionary(conf_key_field))); + } + conv_radius_array = i.conv_radius_array; + conv_thresh_array = i.conv_thresh_array; + merge_thresh_array = i.merge_thresh_array; + conv_thresh = i.conv_thresh; + merge_thresh = i.merge_thresh; + merge_flag = i.merge_flag; + raw_pi = i.raw_pi; + filter_attr_map = i.filter_attr_map; + file_type = i.file_type; + + +} + //////////////////////////////////////////////////////////////////////// @@ -143,6 +175,8 @@ merge_thresh.clear(); merge_flag = MergeType_Engine; +file_type = FileType_None; + // raw_pi.clear(); } @@ -310,8 +344,8 @@ if ( FO == 'F' ) raw_pi = parse_conf_plot_info(conf->lookup_dictionary(conf_key else raw_pi = parse_conf_plot_info(conf->lookup_dictionary(conf_key_obs_raw_plot)); - - +file_type = parse_conf_file_type(dict); + // // done // diff --git a/src/libcode/vx_shapedata/mode_field_info.h b/src/libcode/vx_shapedata/mode_field_info.h index 132f2df4b2..bb7e08cb57 100644 --- a/src/libcode/vx_shapedata/mode_field_info.h +++ b/src/libcode/vx_shapedata/mode_field_info.h @@ -42,10 +42,9 @@ class Mode_Field_Info { void assign(const Mode_Field_Info &); + Dictionary * dict; // not allocated, based on reading the config file - Dictionary * dict; // not allocated - - MetConfig * conf; // not allocated + MetConfig * conf; // not allocated, based on reading the config file GrdFileType gft; @@ -59,6 +58,7 @@ class Mode_Field_Info { ~Mode_Field_Info(); Mode_Field_Info(const Mode_Field_Info &); Mode_Field_Info & operator=(const Mode_Field_Info &); + void clone(const Mode_Field_Info &); void clear(); @@ -85,6 +85,8 @@ class Mode_Field_Info { PlotInfo raw_pi; // Raw forecast plotting info + GrdFileType file_type; // each field can have a configured file type + // // member functions // diff --git a/src/libcode/vx_shapedata/mode_input_data.h b/src/libcode/vx_shapedata/mode_input_data.h new file mode 100644 index 0000000000..dd694c2126 --- /dev/null +++ b/src/libcode/vx_shapedata/mode_input_data.h @@ -0,0 +1,45 @@ +// ** Copyright UCAR (c) 1992 - 2023 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + + +#ifndef __MODE_INPUT_DATA_H__ +#define __MODE_INPUT_DATA_H__ + +#include +#include "data_plane.h" +#include "grid_base.h" +#include "data_file_type.h" + +//////////////////////////////////////////////////////////////////////// + + +class ModeInputData { + + private: + + public: + + ModeInputData(const std::string &name, const DataPlane &dp, + const Grid &g, const GrdFileType t) : + _name(name), _dataPlane(dp), _grid(g), _fileType(t) { } + + ~ModeInputData() {} + + std::string _name; + DataPlane _dataPlane; + Grid _grid; + GrdFileType _fileType; +}; + + +#endif + + +///////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/mode/Makefile.am b/src/tools/core/mode/Makefile.am index 72a35a6175..59e9c74937 100644 --- a/src/tools/core/mode/Makefile.am +++ b/src/tools/core/mode/Makefile.am @@ -13,6 +13,7 @@ include ${top_srcdir}/Make-include bin_PROGRAMS = mode mode_SOURCES = mode_usage.cc \ mode_frontend.cc \ + mode_superobject.cc \ multivar_data.cc \ multivar_frontend.cc \ mode.cc \ diff --git a/src/tools/core/mode/combine_boolplanes.h b/src/tools/core/mode/combine_boolplanes.h index e879827740..084a31aed1 100644 --- a/src/tools/core/mode/combine_boolplanes.h +++ b/src/tools/core/mode/combine_boolplanes.h @@ -20,7 +20,8 @@ #include "two_d_array.h" #include "bool_calc.h" #include "vx_pxm.h" - +#include +#include //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/mode/mode.cc b/src/tools/core/mode/mode.cc index b7277b0110..68463a6c84 100644 --- a/src/tools/core/mode/mode.cc +++ b/src/tools/core/mode/mode.cc @@ -192,7 +192,7 @@ int met_main(int argc, char * argv []) } else { - // run the traditional version of mode + // run the traditional version of mode, with command line arguments as is ModeFrontEnd *frontend = new ModeFrontEnd; status = frontend->run_traditional(Argv); diff --git a/src/tools/core/mode/mode_exec.cc b/src/tools/core/mode/mode_exec.cc index 82cb261b11..28818ba8b8 100644 --- a/src/tools/core/mode/mode_exec.cc +++ b/src/tools/core/mode/mode_exec.cc @@ -180,7 +180,7 @@ void ModeExecutive::init_traditional(int n_files) otype = obs_mtddf->file_type(); // Process the configuration - engine.conf_info.process_config(ftype, otype); + engine.conf_info.process_config_traditional(ftype, otype); int nf = engine.conf_info.n_fields_f(); // same as obs for traditional mode if (nf != n_files) { mlog << Error << "\nNumber of input files " << n_files << " Not equal to config number of fields " @@ -207,56 +207,16 @@ void ModeExecutive::init_traditional(int n_files) /////////////////////////////////////////////////////////////////////// -void ModeExecutive::init_multivar_verif_grid() - +void ModeExecutive::init_multivar_verif_grid(const DataPlane &fcst, + const DataPlane &obs, + const ModeConfInfo &conf) { - Met2dDataFileFactory mtddf_factory; - R_index = T_index = 0; - - conf_read(default_multivar_config_filename); - - // Get the forecast and observation file types from config, if present - ftype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_fcst)); - otype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_obs)); - - - // Read observation file - if(!(obs_mtddf = mtddf_factory.new_met_2d_data_file(obs_file.c_str(), otype))) { - mlog << Error << "\nTrouble reading observation file \"" - << obs_file << "\"\n\n"; - exit(1); - } - - // Read forecast file - if(!(fcst_mtddf = mtddf_factory.new_met_2d_data_file(fcst_file.c_str(), ftype))) { - mlog << Error << "\nTrouble reading forecast file \"" - << fcst_file << "\"\n\n"; - exit(1); - } - - // Store the input data file types - ftype = fcst_mtddf->file_type(); - otype = obs_mtddf->file_type(); - - // Process the configuration - engine.conf_info.process_config(ftype, otype, ModeDataType_MvMode_Both); - - // check one again for multivar problems + engine.conf_info = conf; engine.conf_info.check_multivar_not_implemented(); - - const int shift = engine.conf_info.shift_right; - - fcst_mtddf->set_shift_right(shift); - obs_mtddf->set_shift_right(shift); - - // List the input files - mlog << Debug(1) - << "Forecast File: " << fcst_file << "\n" - << "Observation File: " << obs_file << "\n"; - engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); + // engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); return; @@ -264,103 +224,19 @@ void ModeExecutive::init_multivar_verif_grid() /////////////////////////////////////////////////////////////////////// -void ModeExecutive::init_multivar_simple(int n_files, ModeDataType dtype) +void ModeExecutive::init_multivar_simple(int j, int n_files, ModeDataType dtype, + const ModeConfInfo &conf) { - - Met2dDataFileFactory mtddf_factory; - R_index = T_index = 0; - - conf_read(default_multivar_config_filename); + engine.conf_info = conf; // tell the engine which type of data it is engine.set_data_type(dtype); - // Get the forecast and observation file types from config, if present - ftype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_fcst)); - otype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_obs)); - - - // Read observation file or forecast file - if (dtype == ModeDataType_MvMode_Obs) - { - if(!(obs_mtddf = mtddf_factory.new_met_2d_data_file(obs_file.c_str(), otype))) { - mlog << Error << "\nTrouble reading observation file \"" - << obs_file << "\"\n\n"; - exit(1); - - } - // Store the input data file types - otype = obs_mtddf->file_type(); - } - else if (dtype == ModeDataType_MvMode_Fcst) - { - // Read forecast file - if(!(fcst_mtddf = mtddf_factory.new_met_2d_data_file(fcst_file.c_str(), ftype))) { - mlog << Error << "\nTrouble reading forecast file \"" - << fcst_file << "\"\n\n"; - exit(1); - } - // Store the input data file types - ftype = fcst_mtddf->file_type(); - } - else { - mlog << Error << "\nModeExecutive::init_multivar_simple()->" - << "simple object creation requires obs or forecast mode, got " - << sprintModeDataType(dtype) << "\n\n"; - exit(1); - } - - // Process the configuration - engine.conf_info.process_config(ftype, otype, dtype); - int nf; - if (dtype == ModeDataType_MvMode_Obs) { - nf = engine.conf_info.n_fields_o(); - } else if (dtype == ModeDataType_MvMode_Fcst) { - nf = engine.conf_info.n_fields_f(); - } else { - mlog << Error << "\nModeExecutive::init_multivar_simple()->" - << "simple object creation requires obs or forecast mode, got " - << sprintModeDataType(dtype) << "\n\n"; - exit(1); - } - if (nf != n_files) { - mlog << Error << "\nModeExecutive::init_multivar_simple()->" - << "Number of input files " << n_files << " Not equal to config number of fields " - << nf << "\n\n"; - exit(1); - } - - // check one again for multivar problems - engine.conf_info.check_multivar_not_implemented(); - - const int shift = engine.conf_info.shift_right; - - if (dtype != ModeDataType_MvMode_Fcst) { - obs_mtddf->set_shift_right(shift); - } - - if (dtype != ModeDataType_MvMode_Obs) { - fcst_mtddf->set_shift_right(shift); - } - - if (dtype == ModeDataType_MvMode_Obs) { - mlog << Debug(1) - << "Observation File: " << obs_file << "\n"; + engine.conf_info.set_field_index(j); - } else if (dtype == ModeDataType_MvMode_Fcst) { - mlog << Debug(1) - << "Forecast File: " << fcst_file << "\n"; - } - else { - // List the input files - mlog << Debug(1) - << "Forecast File: " << fcst_file << "\n" - << "Observation File: " << obs_file << "\n"; - } - - engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); + // engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); return; @@ -368,48 +244,27 @@ void ModeExecutive::init_multivar_simple(int n_files, ModeDataType dtype) /////////////////////////////////////////////////////////////////////// -void ModeExecutive::init_multivar_intensities(GrdFileType ftype, GrdFileType otype) +void ModeExecutive::init_multivar_intensities(GrdFileType ftype, GrdFileType otype, const ModeConfInfo &conf) { R_index = T_index = 0; - conf_read(default_multivar_config_filename); - - // Get the forecast and observation file types from config, if present - GrdFileType l_ftype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_fcst)); - GrdFileType l_otype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_obs)); - - // actually use the values passed in instead, the config ones are not set for this second pass - l_ftype = ftype; - l_otype = otype; - - // Process the configuration - engine.conf_info.process_config(l_ftype, l_otype, ModeDataType_MvMode_Both); + engine.conf_info = conf; + + // tell the engine which type of data it is + engine.set_data_type(ModeDataType_MvMode_Both); // check one again for multivar problems engine.conf_info.check_multivar_not_implemented(); - // NOTE: do not do shifting, should have been done in the first pass - // const int shift = engine.conf_info.shift_right; - // fcst_mtddf->set_shift_right(shift); - // obs_mtddf->set_shift_right(shift); - - engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); + // engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); return; } -/////////////////////////////////////////////////////////////////////// - -void ModeExecutive::check_multivar_perc_thresh_settings() -{ - engine.conf_info.check_multivar_perc_thresh(ptype == ModeExecutive::MULTIVAR_SIMPLE, - ptype == ModeExecutive::MULTIVAR_SIMPLE_MERGE); -} - /////////////////////////////////////////////////////////////////////// @@ -569,8 +424,8 @@ void ModeExecutive::setup_fcst_obs_data_traditional() /////////////////////////////////////////////////////////////////////// -void ModeExecutive::setup_verification_grid() - +void ModeExecutive::setup_verification_grid(const ModeInputData &fcst, + const ModeInputData &obs) { // ShapeData fcst_sd, obs_sd; @@ -579,57 +434,29 @@ void ModeExecutive::setup_verification_grid() Fcst_sd.clear(); Obs_sd.clear(); - // Read the gridded data from the input forecast file - - if ( !(fcst_mtddf->data_plane(*(engine.conf_info.Fcst->var_info), Fcst_sd.data)) ) { - - mlog << Error << "\nModeExecutive::setup_verification_grid() -> " - << "can't get forecast data \"" - << engine.conf_info.Fcst->var_info->magic_str() - << "\" from file \"" << fcst_file << "\"\n\n"; - exit(1); - } - - // Read the gridded data from the input observation file - - if ( !(obs_mtddf->data_plane(*(engine.conf_info.Obs->var_info), Obs_sd.data)) ) { - - mlog << Error << "\nModeExecutive::setup_verification_grid() -> " - << "can't get observation data \"" - << engine.conf_info.Obs->var_info->magic_str() - << "\" from file \"" << obs_file << "\"\n\n"; - exit(1); - } + Fcst_sd.data = fcst._dataPlane; + Obs_sd.data = obs._dataPlane; // Determine the verification grid + engine.conf_info.set_field_index(0); grid = parse_vx_grid(engine.conf_info.Fcst->var_info->regrid(), - &(fcst_mtddf->grid()), &(obs_mtddf->grid())); - + &fcst._grid, &obs._grid); return; } /////////////////////////////////////////////////////////////////////// -void ModeExecutive::setup_fcst_data(const Grid &verification_grid) - +void ModeExecutive::setup_fcst_data(const Grid &verification_grid, + const ModeInputData &input) { - // ShapeData fcst_sd, obs_sd; double fmin, fmax; Fcst_sd.clear(); - - // Read the gridded data from the input forecast file - if ( !(fcst_mtddf->data_plane(*(engine.conf_info.Fcst->var_info), Fcst_sd.data)) ) { - - mlog << Error << "\nModeExecutive::setup_fcst_data() -> " - << "can't get forecast data \"" - << engine.conf_info.Fcst->var_info->magic_str() - << "\" from file \"" << fcst_file << "\"\n\n"; - exit(1); - } + ftype = input._fileType; + Fcst_sd.data = input._dataPlane; grid = verification_grid; @@ -638,12 +465,11 @@ void ModeExecutive::setup_fcst_data(const Grid &verification_grid) engine.set_grid(&grid); // Regrid, if necessary - - if ( !(fcst_mtddf->grid() == grid) ) { + if ( !(input._grid == grid) ) { mlog << Debug(1) << "Regridding forecast " << engine.conf_info.Fcst->var_info->magic_str() << " to the verification grid.\n"; - Fcst_sd.data = met_regrid(Fcst_sd.data, fcst_mtddf->grid(), grid, + Fcst_sd.data = met_regrid(Fcst_sd.data, input._grid, grid, engine.conf_info.Fcst->var_info->regrid()); } @@ -667,10 +493,6 @@ void ModeExecutive::setup_fcst_data(const Grid &verification_grid) data_min = fmin; data_max = fmax; - // Process percentile thresholds - - engine.conf_info.set_perc_thresh(Fcst_sd.data); - // store the input data units funits = engine.conf_info.Fcst->var_info->units(); ounits = na_str; @@ -690,25 +512,18 @@ void ModeExecutive::setup_fcst_data(const Grid &verification_grid) /////////////////////////////////////////////////////////////////////// -void ModeExecutive::setup_obs_data(const Grid &verification_grid) - +void ModeExecutive::setup_obs_data(const Grid &verification_grid, + const ModeInputData &input) { // ShapeData fcst_sd, obs_sd; double omin, omax; Obs_sd.clear(); + otype = input._fileType; // Read the gridded data from the input observation file - - if ( !(obs_mtddf->data_plane(*(engine.conf_info.Obs->var_info), Obs_sd.data)) ) { - - mlog << Error << "\nModeExecutive::setup_obs_data() -> " - << "can't get observation data \"" - << engine.conf_info.Obs->var_info->magic_str() - << "\" from file \"" << obs_file << "\"\n\n"; - exit(1); - } + Obs_sd.data = input._dataPlane; grid = verification_grid; @@ -718,11 +533,11 @@ void ModeExecutive::setup_obs_data(const Grid &verification_grid) // Regrid, if necessary - if ( !(obs_mtddf->grid() == grid) ) { + if ( !(input._grid == grid) ) { mlog << Debug(1) << "Regridding observation " << engine.conf_info.Obs->var_info->magic_str() << " to the verification grid.\n"; - Obs_sd.data = met_regrid(Obs_sd.data, obs_mtddf->grid(), grid, + Obs_sd.data = met_regrid(Obs_sd.data, input._grid, grid, engine.conf_info.Obs->var_info->regrid()); } @@ -746,10 +561,6 @@ void ModeExecutive::setup_obs_data(const Grid &verification_grid) data_min = omin; data_max = omax; - // Process percentile thresholds - - engine.conf_info.set_perc_thresh(Obs_sd.data); - // store the input data units funits = na_str; ounits = engine.conf_info.Obs->var_info->units(); @@ -780,17 +591,11 @@ void ModeExecutive::setup_fcst_obs_data_multivar_intensities(const MultiVarData Obs_sd.debug_examine(); grid = *(mvdf._grid); - // do not need to read any data, it is stored in the mvd input - // the verification grid was created in the first pass, so we have that as well - // Store the grid engine.set_grid(&grid); - // regridding of inputs is not needed in the second pass, as regridded data outputs - // are stored to input mvd - - // Rescale probabilites from [0, 100] to [0, 1] also not neede in the second pass + // Rescale probabilites from [0, 100] to [0, 1] not needed in the second pass // Print a warning if the valid times do not match if(Fcst_sd.data.valid() != Obs_sd.data.valid()) { @@ -835,15 +640,12 @@ void ModeExecutive::setup_fcst_obs_data_multivar_intensities(const MultiVarData // in case perc thresh was done, retrieve the values created in the // first pass, which have been stored in mvdo and mvdf + // NOTE: this might be removable with new design engine.conf_info.Obs->conv_thresh_array = mvdo._merge->_convThreshArray; engine.conf_info.Obs->merge_thresh_array = mvdo._merge->_mergeThreshArray; engine.conf_info.Fcst->conv_thresh_array = mvdf._merge->_convThreshArray; engine.conf_info.Fcst->merge_thresh_array = mvdf._merge->_mergeThreshArray; - // This doesn't work with perc_thresh, it uses values restricted to within - // superobjects, which gives different thresholds, so we do the above instead. - //engine.conf_info.set_perc_thresh(Fcst_sd.data, Obs_sd.data); - // masking of inputs not needed, as it was done in the first pass // and stored to Fcst_sd and Obs_sd @@ -867,7 +669,8 @@ void ModeExecutive::setup_fcst_obs_data_multivar_intensities(const MultiVarData /////////////////////////////////////////////////////////////////////// -void ModeExecutive::setup_fcst_obs_data_multivar_super(ShapeData &f_super, ShapeData &o_super, +void ModeExecutive::setup_fcst_obs_data_multivar_super(const ShapeData &f_super, + const ShapeData &o_super, const Grid &igrid) { double fmin, omin, fmax, omax; @@ -941,10 +744,6 @@ void ModeExecutive::setup_fcst_obs_data_multivar_super(ShapeData &f_super, Shape else if(!is_bad_data(fmax) && is_bad_data(omax)) data_max = fmax; else if( is_bad_data(fmax) && !is_bad_data(omax)) data_max = omax; - // Process percentile thresholds...Not applicable for this pass - // do this in the second pass using masked Fcst_sd and Obs_sd - // engine.conf_info.set_perc_thresh(Fcst_sd.data, Obs_sd.data); - // // done // @@ -1074,7 +873,7 @@ void ModeExecutive::do_merging() /////////////////////////////////////////////////////////////////////// -void ModeExecutive::do_merging(ShapeData &f_merge, ShapeData &o_merge) +void ModeExecutive::do_merging(const ShapeData &f_merge, const ShapeData &o_merge) { if (ptype == MULTIVAR_SUPER) { @@ -1127,7 +926,8 @@ void ModeExecutive::do_merging(ShapeData &f_merge, ShapeData &o_merge) /////////////////////////////////////////////////////////////////////// -void ModeExecutive::do_match_merge(ShapeData &f_merge, ShapeData &o_merge) +void ModeExecutive::do_match_merge(const ShapeData &f_merge, + const ShapeData &o_merge) { do_merging(f_merge, o_merge); diff --git a/src/tools/core/mode/mode_exec.h b/src/tools/core/mode/mode_exec.h index 34c8d7fb5e..c326c6ab30 100644 --- a/src/tools/core/mode/mode_exec.h +++ b/src/tools/core/mode/mode_exec.h @@ -81,10 +81,10 @@ class ModeExecutive { void clear(); void init_traditional(int n_files); - void init_multivar_verif_grid(); - void init_multivar_simple(int n_files, ModeDataType dtype); - void init_multivar_intensities(GrdFileType ftype, GrdFileType otype); - void check_multivar_perc_thresh_settings(); + void init_multivar_verif_grid(const DataPlane &fcst, + const DataPlane &obs, const ModeConfInfo &config); + void init_multivar_simple(int j, int n_files, ModeDataType dtype, const ModeConfInfo &conf); + void init_multivar_intensities(GrdFileType ftype, GrdFileType otype, const ModeConfInfo &conf); int n_conv_radii () const; @@ -135,17 +135,20 @@ class ModeExecutive { Processing_t ptype; void clear_internal_r_index(); - void setup_verification_grid(); + void setup_verification_grid(const ModeInputData &fcst, + const ModeInputData &obs); void setup_fcst_obs_data_traditional(); - void setup_fcst_data(const Grid &verification_grid); - void setup_obs_data(const Grid &verification_grid); - void setup_fcst_obs_data_multivar_intensities(const MultiVarData &mvdf, const MultiVarData &mvdo); - void setup_fcst_obs_data_multivar_super(ShapeData &f_super, ShapeData &o_super, const Grid &igrid); + void setup_fcst_data(const Grid &verification_grid, const ModeInputData &input); + void setup_obs_data(const Grid &verification_grid, const ModeInputData &input); + void setup_fcst_obs_data_multivar_intensities(const MultiVarData &mvdf, + const MultiVarData &mvdo); + void setup_fcst_obs_data_multivar_super(const ShapeData &f_super, + const ShapeData &o_super, const Grid &igrid); void do_conv_thresh(const int r_index, const int t_index); void do_merging(); - void do_merging(ShapeData &f_merge, ShapeData &o_merge); + void do_merging(const ShapeData &f_merge, const ShapeData &o_merge); void do_match_merge(); - void do_match_merge(ShapeData &f_merge, ShapeData &o_merge); + void do_match_merge(const ShapeData &f_merge, const ShapeData &o_merge); void process_masks(ShapeData &, ShapeData &); void process_fcst_masks(ShapeData &); diff --git a/src/tools/core/mode/mode_frontend.cc b/src/tools/core/mode/mode_frontend.cc index 456ae71f7a..efbe774485 100644 --- a/src/tools/core/mode/mode_frontend.cc +++ b/src/tools/core/mode/mode_frontend.cc @@ -36,6 +36,9 @@ extern const char * const program_name; static ModeExecutive *mode_exec = 0; static ModeExecutive::Processing_t ptype = ModeExecutive::TRADITIONAL; + +// used only for traditional mode, multivar sets it into config previous +// to the frontend creation static int compress_level = -1; @@ -63,31 +66,22 @@ ModeFrontEnd::~ModeFrontEnd() /////////////////////////////////////////////////////////////////////// - -Grid ModeFrontEnd::create_verification_grid(const StringArray & Argv) - +Grid ModeFrontEnd::create_verification_grid(const ModeInputData &fcst, + const ModeInputData &obs, + const string &config_file, + const ModeConfInfo &config) { if ( mode_exec ) { delete mode_exec; mode_exec = 0; } mode_exec = new ModeExecutive; - compress_level = -1; - - // - // Process the command line arguments - // - - process_command_line(Argv, false); - - mode_exec->init_multivar_verif_grid(); + mode_exec->out_dir = default_out_dir; + mode_exec->fcst_file = "None"; + mode_exec->obs_file = "None"; + mode_exec->match_config_file = config_file; + mode_exec->init_multivar_verif_grid(fcst._dataPlane, obs._dataPlane, config); ModeConfInfo & conf = mode_exec->engine.conf_info; conf.set_field_index(0); - if (compress_level >= 0) conf.nc_info.set_compress_level(compress_level); - - - // - // read in data (Note multiple reads of same data) - // - mode_exec->setup_verification_grid(); + mode_exec->setup_verification_grid(fcst, obs); Grid g = mode_exec->grid; delete mode_exec; mode_exec = 0; return g; @@ -96,35 +90,38 @@ Grid ModeFrontEnd::create_verification_grid(const StringArray & Argv) /////////////////////////////////////////////////////////////////////// -int ModeFrontEnd::create_multivar_simple_objects(const StringArray & Argv, ModeDataType dtype, - const Grid &verification_grid, int field_index, int n_files) +int +ModeFrontEnd::create_multivar_simple_objects( + const ModeConfInfo &config, + ModeDataType dtype, + const Grid &verification_grid, + const ModeInputData &input, + const string &filename, + const string &config_file, + const string &outdir, + int field_index, int n_files) { init(ModeExecutive::MULTIVAR_SIMPLE); - // - // Process the command line arguments - // - - process_command_line_for_simple_objects(Argv, dtype); - - mode_exec->init_multivar_simple(n_files, dtype); + mode_exec->out_dir = replace_path(default_out_dir); + if (dtype == ModeDataType_MvMode_Fcst) { + mode_exec->fcst_file = filename; + mode_exec->obs_file = "None"; + } else { + mode_exec->obs_file = filename; + mode_exec->fcst_file = "None"; + } + mode_exec->match_config_file = config_file; + mode_exec->out_dir = outdir; + mode_exec->init_multivar_simple(field_index, n_files, dtype, config); ModeConfInfo & conf = mode_exec->engine.conf_info; - if ( field_index >= 0 ) conf.set_field_index(field_index); - if (compress_level >= 0) conf.nc_info.set_compress_level(compress_level); - - // need to do this after setting field index above - mode_exec->check_multivar_perc_thresh_settings(); - - // - // read in data (Note multiple reads of data) - // if (dtype == ModeDataType_MvMode_Fcst) { - mode_exec->setup_fcst_data(verification_grid); + mode_exec->setup_fcst_data(verification_grid, input); } else { - mode_exec->setup_obs_data(verification_grid); + mode_exec->setup_obs_data(verification_grid, input); } // @@ -152,37 +149,37 @@ int ModeFrontEnd::create_multivar_simple_objects(const StringArray & Argv, ModeD /////////////////////////////////////////////////////////////////////// -int ModeFrontEnd::create_multivar_merge_objects(const StringArray & Argv, ModeDataType dtype, - const Grid &verification_grid, int field_index, - int n_files) - +int +ModeFrontEnd::create_multivar_merge_objects(const ModeConfInfo &config, + ModeDataType dtype, + const Grid &verification_grid, + const ModeInputData &input, + const string &filename, + const string &config_file, + const string &outdir, int field_index, + int n_files) { init(ModeExecutive::MULTIVAR_SIMPLE_MERGE); - // - // Process the command line arguments - // - - process_command_line_for_simple_objects(Argv, dtype); + mode_exec->out_dir = replace_path(default_out_dir); + if (dtype == ModeDataType_MvMode_Fcst) { + mode_exec->fcst_file = filename; + mode_exec->obs_file = "None"; + } else { + mode_exec->obs_file = filename; + mode_exec->fcst_file = "None"; + } + mode_exec->match_config_file = config_file; + mode_exec->out_dir = outdir; - mode_exec->init_multivar_simple(n_files, dtype); + mode_exec->init_multivar_simple(field_index, n_files, dtype, config); ModeConfInfo & conf = mode_exec->engine.conf_info; - if ( field_index >= 0 ) conf.set_field_index(field_index); - if (compress_level >= 0) conf.nc_info.set_compress_level(compress_level); - - // need to do this after setting field index above - mode_exec->check_multivar_perc_thresh_settings(); - - - // - // read in data (Note multiple reads not desired) - // if (dtype == ModeDataType_MvMode_Fcst) { - mode_exec->setup_fcst_data(verification_grid); + mode_exec->setup_fcst_data(verification_grid, input); } else { - mode_exec->setup_obs_data(verification_grid); + mode_exec->setup_obs_data(verification_grid, input); } @@ -263,23 +260,28 @@ int ModeFrontEnd::run_traditional(const StringArray & Argv) /////////////////////////////////////////////////////////////////////// -int ModeFrontEnd::multivar_intensity_comparisons(const StringArray & Argv, const MultiVarData &mvdf, - const MultiVarData &mvdo, bool has_union_f, - bool has_union_o, ShapeData &merge_f, - ShapeData &merge_o, int field_index_f, int field_index_o) +int +ModeFrontEnd::multivar_intensity_comparisons(const ModeConfInfo &config, + const MultiVarData &mvdf, + const MultiVarData &mvdo, bool has_union_f, + bool has_union_o, const ShapeData &merge_f, + const ShapeData &merge_o, + int field_index_f, int field_index_o, + const string &fcst_filename, + const string &obs_filename, + const string &config_file, const string &dir) { init(ModeExecutive::MULTIVAR_INTENSITY); - // - // Process the command line arguments - // - - process_command_line(Argv, false); - - mode_exec->init_multivar_intensities(mvdf._type, mvdo._type); + mode_exec->out_dir = replace_path(default_out_dir); + mode_exec->fcst_file = fcst_filename; + mode_exec->obs_file = obs_filename; + mode_exec->match_config_file = config_file; + mode_exec->out_dir = dir; + + mode_exec->init_multivar_intensities(mvdf._type, mvdo._type, config); ModeConfInfo & conf = mode_exec->engine.conf_info; - if (compress_level >= 0) conf.nc_info.set_compress_level(compress_level); conf.set_field_index(field_index_f, field_index_o); // for multivar intensities, explicity set the level and units using stored values @@ -330,24 +332,23 @@ int ModeFrontEnd::multivar_intensity_comparisons(const StringArray & Argv, const /////////////////////////////////////////////////////////////////////// -int ModeFrontEnd::run_super(const StringArray & Argv, - ShapeData &f_super, ShapeData &o_super, - ShapeData &f_merge, ShapeData &o_merge, +int ModeFrontEnd::run_super(const ModeConfInfo &config, + const ModeSuperObject &fsuper, + const ModeSuperObject &osuper, GrdFileType ftype, GrdFileType otype, const Grid &grid, - bool has_union) + bool has_union, const string &config_file, const string &dir) { init(ModeExecutive::MULTIVAR_SUPER); - // - // Process the command line arguments - // - - process_command_line(Argv, true); + mode_exec->out_dir = replace_path(default_out_dir); + mode_exec->fcst_file = "not set"; + mode_exec->obs_file = "not set"; + mode_exec->match_config_file = config_file; + mode_exec->out_dir = dir; - mode_exec->init_multivar_intensities(ftype, otype); + mode_exec->init_multivar_intensities(ftype, otype, config); ModeConfInfo & conf = mode_exec->engine.conf_info; - if (compress_level >= 0) conf.nc_info.set_compress_level(compress_level); if (has_union && (conf.Fcst->merge_flag == MergeType_Thresh || conf.Obs->merge_flag == MergeType_Thresh)) { mlog << Warning << "\nModeFrontEnd::run_super() -> " @@ -358,7 +359,7 @@ int ModeFrontEnd::run_super(const StringArray & Argv, // // set up data access using inputs // - mode_exec->setup_fcst_obs_data_multivar_super(f_super, o_super, grid); + mode_exec->setup_fcst_obs_data_multivar_super(fsuper._simple_sd, osuper._simple_sd, grid); // // run the mode algorithm @@ -370,7 +371,7 @@ int ModeFrontEnd::run_super(const StringArray & Argv, } else { - do_straight_multivar_super(f_merge, o_merge); + do_straight_multivar_super(fsuper._merge_sd_split, osuper._merge_sd_split); } @@ -421,8 +422,8 @@ void ModeFrontEnd::do_straight() void ModeFrontEnd::do_straight_multivar_intensity(const MultiVarData &mvdf, const MultiVarData &mvdo, - ShapeData &f_merge, - ShapeData &o_merge) + const ShapeData &f_merge, + const ShapeData &o_merge) { int NCT, NCR; @@ -456,7 +457,8 @@ void ModeFrontEnd::do_straight_multivar_intensity(const MultiVarData &mvdf, /////////////////////////////////////////////////////////////////////// -void ModeFrontEnd::do_straight_multivar_super(ShapeData &f_merge, ShapeData &o_merge) +void ModeFrontEnd::do_straight_multivar_super(const ShapeData &f_merge, + const ShapeData &o_merge) { int NCT, NCR; diff --git a/src/tools/core/mode/mode_frontend.h b/src/tools/core/mode/mode_frontend.h index 55ecdf7f50..f12b40eaa0 100644 --- a/src/tools/core/mode/mode_frontend.h +++ b/src/tools/core/mode/mode_frontend.h @@ -20,7 +20,9 @@ #include "mode_exec.h" #include "string_array.h" #include "multivar_data.h" +#include "mode_input_data.h" #include "mode_data_type.h" +#include "mode_superobject.h" class ModeFrontEnd { @@ -34,14 +36,26 @@ class ModeFrontEnd { string default_out_dir; - Grid create_verification_grid(const StringArray & Argv); + Grid create_verification_grid(const ModeInputData &fcst, + const ModeInputData &obs, + const string &config_file, + const ModeConfInfo &config); + // run the multivar simple object, where there is only one input data, either forecast or obs - int create_multivar_simple_objects(const StringArray & Argv, ModeDataType dtype, const Grid &verification_grid, + int create_multivar_simple_objects(const ModeConfInfo &conf, + ModeDataType dtype, const Grid &verification_grid, + const ModeInputData &input, + const string &filename, const string &config_file, + const string &outdir, int field_index=-1, int n_files=1); // run the multivar simple object merge algorithm, with one input data, either forecast or obs - int create_multivar_merge_objects(const StringArray & Argv, ModeDataType dtype, const Grid &verification_grid, + int create_multivar_merge_objects(const ModeConfInfo &conf, + ModeDataType dtype, const Grid &verification_grid, + const ModeInputData &input, + const string &filename, const string &config_file, + const string &outdir, int field_index=-1, int n_files=1); // run the default single var mode interface (traditional mode) @@ -49,15 +63,22 @@ class ModeFrontEnd { // run the multivar intensity algorithm, where one forecast and one obs are restricted to be within superobjects // and the traditional mode algorithm compares them - int multivar_intensity_comparisons(const StringArray & Argv, const MultiVarData &mvdf, const MultiVarData &mvdo, - bool has_union_f, bool has_union_o, ShapeData &merge_f, - ShapeData &merge_o, int field_index_f, int field_index_o); - + int multivar_intensity_comparisons(const ModeConfInfo &conf, + const MultiVarData &mvdf, const MultiVarData &mvdo, + bool has_union_f, bool has_union_o, const ShapeData &merge_f, + const ShapeData &merge_o, int field_index_f, int field_index_o, + const string &fcst_filename, const string &obs_filename, + const string &config_file, const string &dir); + + // multivar superobject interface, with no intensities - int run_super(const StringArray & Argv, ShapeData &f_super, ShapeData &o_super, - ShapeData &f_merge, ShapeData &o_merge, - GrdFileType ftype, GrdFileType otype, const Grid &grid, bool has_union); + int run_super(const ModeConfInfo &conf, + const ModeSuperObject &fsuper, + const ModeSuperObject &osuper, + GrdFileType ftype, GrdFileType otype, const Grid &grid, bool has_union, + const string &config_file, const string &dir); + // so far only implemented for traditional mode void do_quilt (); @@ -66,11 +87,13 @@ class ModeFrontEnd { // MODE algorithm when doing multivar intensities void do_straight_multivar_intensity (const MultiVarData &mvdf, - const MultiVarData &mvdo, ShapeData &mergef, - ShapeData &mergeo); + const MultiVarData &mvdo, + const ShapeData &mergef, + const ShapeData &mergeo); // MODE algorithm when doing multivar super with no intensities - void do_straight_multivar_super (ShapeData &f_merge, ShapeData &o_merge); + void do_straight_multivar_super (const ShapeData &f_merge, + const ShapeData &o_merge); MultiVarData *get_multivar_data(ModeDataType dtype); diff --git a/src/tools/core/mode/mode_superobject.cc b/src/tools/core/mode/mode_superobject.cc new file mode 100644 index 0000000000..c6e24dcff3 --- /dev/null +++ b/src/tools/core/mode/mode_superobject.cc @@ -0,0 +1,102 @@ +using namespace std; +#include "mode_superobject.h" + +//////////////////////////////////////////////////////////////////////// + +static void _debug_shape_examine(string &name, const ShapeData &sd, + int nx, int ny) +{ + vector values; + vector count; + for (int x=0; x::iterator vi; + vi = find(values.begin(), values.end(), v); + if (vi == values.end()) { + values.push_back(v); + count.push_back(1); + } else { + int ii = vi - values.begin(); + count[ii] = count[ii] + 1; + } + } + } + for (size_t i=0; i &mvd, + BoolCalc &calc) +{ + // + // set the BoolPlane values using the mvd content + // + + BoolPlane * simple_plane = new BoolPlane [n_files]; + BoolPlane * merge_plane = new BoolPlane [n_files]; + + for (int j=0; jobjects_from_arrays(do_clusters, true, simple_plane[j]); + mvd[j]->objects_from_arrays(do_clusters, false, merge_plane[j]); + } + + // + // combine the objects into super-objects + // + const int nx = simple_plane[0].nx(); + const int ny = simple_plane[0].ny(); + + BoolPlane merge_result; // local, not used + _simple_result.set_size(nx, ny); + merge_result.set_size(nx, ny); + + string simple_name, merge_name; + + if (isFcst) { + simple_name = "Fcst_Simple"; + merge_name = "Fcst_Merge"; + } else { + simple_name = "Obs_Simple"; + merge_name = "Obs_Merge"; + } + combine_boolplanes(simple_name, simple_plane, n_files, calc, _simple_result); + combine_boolplanes(merge_name, merge_plane, n_files, calc, merge_result); + + // create ShapeData objects using something from mvd as a template + // (shape data has 1's or bad) + + _simple_sd = ShapeData(*(mvd[0]->_simple->_sd)); + for (int x=0; x_simple->_sd)); + for (int x=0; x +#include + +class ModeSuperObject { + + private: + + public: + + ModeSuperObject(bool isFcst, int n_files, bool do_clusters, + const vector &mvd, + BoolCalc &calc); + inline ~ModeSuperObject() {} + + bool _isFcst; + BoolPlane _simple_result; + ShapeData _simple_sd; + ShapeData _merge_sd_split; +}; + + +#endif /* __MODE_SUPEROBJECT_H__ */ + +///////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/mode/multivar_frontend.cc b/src/tools/core/mode/multivar_frontend.cc index 2205fc4329..1512bc8a83 100644 --- a/src/tools/core/mode/multivar_frontend.cc +++ b/src/tools/core/mode/multivar_frontend.cc @@ -51,6 +51,7 @@ using namespace std; #include "parse_file_list.h" #include "mode_frontend.h" #include "multivar_data.h" +#include "mode_input_data.h" #include "mode_data_type.h" using namespace netCDF; @@ -67,7 +68,6 @@ static const char sep [] = "==================================================== static const char tab [] = " "; // this is hardwired for the multivar case, at least for now - static const bool do_clusters = false; static string default_out_dir = "."; @@ -79,6 +79,7 @@ static string fcst_fof; static string obs_fof; static string config_file; static string outdir; +static int compress_level = -1; static Grid verification_grid; @@ -88,6 +89,11 @@ static Grid verification_grid; static void set_outdir (const StringArray &); static void set_logfile (const StringArray &); static void set_verbosity (const StringArray &); +static void set_compress (const StringArray &); + +static void read_input(const string &name, int index, ModeDataType type, + GrdFileType f_t, GrdFileType other_t, int shift, + vector &inputs); static void multivar_consistency_checks(StringArray &fcst_filenames, StringArray &obs_filenames, BoolCalc &f_calc, BoolCalc &o_calc, int &n_fcst_files, @@ -95,35 +101,31 @@ static void multivar_consistency_checks(StringArray &fcst_filenames, StringArray static ConcatString set_multivar_dir(); -static void create_verification_grid(const string &fcst_filename, - const string &obs_filename, - const ConcatString &dir); +static void create_verification_grid(const ModeInputData &fcst, const ModeInputData &obs); + static MultiVarData *create_simple_objects(ModeDataType dtype, int j, int n_files, const string &filename, - const ConcatString &dir); - -static void create_superobjects(int n_fcst_files, const vector &mvdFcst, - int n_obs_files, const vector &mvdObs, - BoolCalc &f_calc, BoolCalc &o_calc, - BoolPlane &f_simple_result, BoolPlane &o_simple_result, - ShapeData &f_simple_sd, ShapeData &o_simple_sd, - ShapeData &f_merge_sd_split, ShapeData &o_merge_sd_split); + const ConcatString &dir, + const ModeInputData &input); -static void create_intensity_comparisons(int findex, int oindex, const BoolPlane &f_result, - BoolPlane &o_result, +static void create_intensity_comparisons(int findex, int oindex, + const ModeSuperObject &fsuper, + const ModeSuperObject &osuper, const ConcatString &dir, MultiVarData &mvdf, MultiVarData &mvdo, bool has_union_f, bool has_union_o, - const string &fcst_filename, const string &obs_filename, - ShapeData &merge_f, ShapeData &merge_o); + const string &fcst_filename, + const string &obs_filename); -static void process_superobjects(ShapeData &f_result, ShapeData &o_result, - ShapeData &f_merge, ShapeData &o_merge, +static void process_superobjects(ModeSuperObject &fsuper, + ModeSuperObject &osuper, int nx, int ny, const ConcatString &dir, - GrdFileType ftype, GrdFileType otype, const Grid &grid, bool has_union); + GrdFileType ftype, GrdFileType otype, const Grid &grid, + bool has_union); -static void mask_data(const string &name, int nx, int ny, const BoolPlane &mask, DataPlane &data); +static void mask_data(const string &name, int nx, int ny, const BoolPlane &mask, + DataPlane &data); static void mask_data_super(const string &name, int nx, int ny, DataPlane &data); static void read_config(const string & filename); @@ -132,9 +134,6 @@ static void process_command_line(const StringArray &); static int _mkdir(const char *dir); -static void _debug_shape_examine(string &name, const ShapeData &sd, int nx, int ny); - - //////////////////////////////////////////////////////////////////////// @@ -151,25 +150,55 @@ int multivar_frontend(const StringArray & Argv) StringArray obs_filenames; BoolCalc f_calc, o_calc ; + // set some logging related things here, used in all further processing + process_command_line(Argv); + // read the config as fully as possible without any data reads + // (Initialize all the input fields) + read_config(config_file); + // check for length discrepencies. + multivar_consistency_checks(fcst_filenames, obs_filenames, f_calc, o_calc, n_fcst_files, n_obs_files); - bool f_has_union = f_calc.has_union(); - bool o_has_union = o_calc.has_union(); - mlog << Debug(2) << "\n" << sep << "\n"; ConcatString dir = set_multivar_dir(); + // read in all the data + vector fcstInput, obsInput; + GrdFileType ft, ot; + + + // in the conf object, shift *can* be set independently for obs and fcst + int shift = config.shift_right; + + for (int i=0; i 0) { mvdFcst[0]->checkFileTypeConsistency(*mvdi, j); } @@ -195,7 +224,7 @@ int multivar_frontend(const StringArray & Argv) mlog << Debug(2) << "\n" << sep << "\ncreating simple obs objects from obs " << (j + 1) << " of " << n_obs_files << "\n" << sep << "\n"; - MultiVarData *mvdi = create_simple_objects(ModeDataType_MvMode_Obs, j, n_obs_files, obs_filenames[j], dir); + MultiVarData *mvdi = create_simple_objects(ModeDataType_MvMode_Obs, j, n_obs_files, obs_filenames[j], dir, obsInput[j]); if (j > 0) { mvdObs[0]->checkFileTypeConsistency(*mvdi, j); } @@ -207,26 +236,25 @@ int multivar_frontend(const StringArray & Argv) // now create forecast and obs superobjects - BoolPlane f_simple_result, o_simple_result; - ShapeData f_simple_sd, o_simple_sd, f_merge_sd_split, o_merge_sd_split; - - create_superobjects(n_fcst_files, mvdFcst, n_obs_files, mvdObs, - f_calc, o_calc, f_simple_result, o_simple_result, - f_simple_sd, o_simple_sd, - f_merge_sd_split, o_merge_sd_split); - + ModeSuperObject fsuper(true, n_fcst_files, do_clusters, mvdFcst, f_calc); + ModeSuperObject osuper(true, n_obs_files, do_clusters, mvdObs, o_calc); + // // Filter the data to within the superobjects only and do statistics by invoking mode algorithm again // on the masked data pairs // + bool f_has_union = f_calc.has_union(); + bool o_has_union = o_calc.has_union(); + for (int k=0; k= 0) config.nc_info.set_compress_level(compress_level); + // from within mode_exec: + // engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); + + return; } @@ -357,6 +403,7 @@ void process_command_line(const StringArray & argv) cline.add(set_outdir, "-outdir", 1); cline.add(set_logfile, "-log", 1); cline.add(set_verbosity, "-v", 1); + cline.add(set_compress, "-compress", 1); cline.parse(); @@ -376,6 +423,42 @@ void process_command_line(const StringArray & argv) //////////////////////////////////////////////////////////////////////// +void read_input(const string &name, int index, ModeDataType type, + GrdFileType f_t, GrdFileType other_t, int shift, + vector &inputs) + { + Met2dDataFileFactory mtddf_factory; + Met2dDataFile *f = mtddf_factory.new_met_2d_data_file(name.c_str(), f_t); + if (!f) { + mlog << Error << "\nTrouble reading fcst file \"" + << name << "\"\n\n"; + exit(1); + } + Grid g = f->grid(); + GrdFileType ft = f->file_type(); + + //? + f->set_shift_right(shift); + + // update config now that we know file type (this sets Fcst to index i) + DataPlane dp; + + if (type == ModeDataType_MvMode_Fcst) { + config.process_config_field(ft, other_t, type, index); + // need to have set the var_info! + f->data_plane(*(config.Fcst->var_info), dp); + } else { + config.process_config_field(other_t, ft, type, index); + // need to have set the var_info! + f->data_plane(*(config.Obs->var_info), dp); + } + + inputs.push_back(ModeInputData(name, dp, g, ft)); + delete f; + } + +//////////////////////////////////////////////////////////////////////// + void multivar_consistency_checks(StringArray &fcst_filenames, StringArray &obs_filenames, BoolCalc &f_calc, BoolCalc &o_calc, int &n_fcst_files, int &n_obs_files) @@ -501,92 +584,28 @@ ConcatString set_multivar_dir() //////////////////////////////////////////////////////////////////////// -void create_verification_grid(const string &fcst_filename, const string &obs_filename, - const ConcatString &dir) +void create_verification_grid(const ModeInputData &fcst, + const ModeInputData &obs) { - ConcatString command; - StringArray a, mode_argv; - - // - // build the command for running mode frontend - // - - mode_argv.clear(); - mode_argv.add(mode_path); - mode_argv.add(fcst_filename); - mode_argv.add(obs_filename); - mode_argv.add(config_file); - - command << cs_erase - << mode_path << ' ' - << fcst_filename << ' ' - << obs_filename << ' ' - << config_file; - - mode_argv.add("-v"); - char junk [256]; - snprintf(junk, sizeof(junk), "%d", mlog.verbosity_level()); - mode_argv.add(junk); - - mode_argv.add("-outdir"); - mode_argv.add(dir); - - command << " -v " << mlog.verbosity_level(); - command << " -outdir " << dir; - - // - // run the pass1 portions of mode, which creates simple objects - // - - mlog << Debug(3) << "Running mode command: \"" << command << "\"\n\n"; ModeFrontEnd *frontend = new ModeFrontEnd; - verification_grid = frontend->create_verification_grid(mode_argv); + verification_grid = frontend->create_verification_grid(fcst, obs, + config_file, + config); delete frontend; - - } //////////////////////////////////////////////////////////////////////// MultiVarData *create_simple_objects(ModeDataType dtype, int j, int n_files, const string &filename, - const ConcatString &dir) + const ConcatString &dir, + const ModeInputData &input) { - ConcatString command; - StringArray a, mode_argv; - - // - // build the command for running mode frontend - // - - mode_argv.clear(); - mode_argv.add(mode_path); - mode_argv.add(filename); - mode_argv.add(config_file); - - command << cs_erase - << mode_path << ' ' - << filename << ' ' - << config_file; - - mode_argv.add("-v"); - char junk [256]; - snprintf(junk, sizeof(junk), "%d", mlog.verbosity_level()); - mode_argv.add(junk); - - mode_argv.add("-outdir"); - mode_argv.add(dir); - - command << " -v " << mlog.verbosity_level(); - command << " -outdir " << dir; - - // - // create the simple objects, forecast or obs, from this input data file - // - - mlog << Debug(3) << "Running mode command: \"" << command << "\"\n\n"; ModeFrontEnd *frontend = new ModeFrontEnd; - int status = frontend->create_multivar_simple_objects(mode_argv, dtype, verification_grid, j, n_files); + int status = + frontend->create_multivar_simple_objects(config, dtype, verification_grid, input, + filename, config_file, dir, + j, n_files); MultiVarData *mvdi = frontend->get_multivar_data(dtype); delete frontend; @@ -595,7 +614,9 @@ MultiVarData *create_simple_objects(ModeDataType dtype, int j, int n_files, // frontend = new ModeFrontEnd; - status = frontend->create_multivar_merge_objects(mode_argv, dtype, verification_grid, j, n_files); + status = frontend->create_multivar_merge_objects(config, dtype, verification_grid, input, + filename, config_file, dir, + j, n_files); // add the merge results to the mvdi object frontend->add_multivar_merge_data(mvdi, dtype); @@ -604,188 +625,62 @@ MultiVarData *create_simple_objects(ModeDataType dtype, int j, int n_files, return mvdi; } -//////////////////////////////////////////////////////////////////////// - -void create_superobjects(int n_fcst_files, const vector &mvdFcst, - int n_obs_files, const vector &mvdObs, - BoolCalc &f_calc, BoolCalc &o_calc, - BoolPlane &f_simple_result, BoolPlane &o_simple_result, - ShapeData &f_simple_sd, ShapeData &o_simple_sd, - ShapeData &f_merge_sd_split, ShapeData &o_merge_sd_split) -{ - // - // set the BoolPlane values using the mvd content - // - - BoolPlane * f_simple_plane = new BoolPlane [n_fcst_files]; - BoolPlane * o_simple_plane = new BoolPlane [n_obs_files]; - BoolPlane * f_merge_plane = new BoolPlane [n_fcst_files]; - BoolPlane * o_merge_plane = new BoolPlane [n_obs_files]; - - for (int j=0; jobjects_from_arrays(do_clusters, true, f_simple_plane[j]); - mvdFcst[j]->objects_from_arrays(do_clusters, false, f_merge_plane[j]); - } - - for (int j=0; jobjects_from_arrays(do_clusters, true, o_simple_plane[j]); - mvdObs[j]->objects_from_arrays(do_clusters, false, o_merge_plane[j]); - } - - // - // combine the objects into super-objects - // - const int nx = f_simple_plane[0].nx(); - const int ny = f_simple_plane[0].ny(); - - BoolPlane f_merge_result, o_merge_result; - f_simple_result.set_size(nx, ny); - o_simple_result.set_size(nx, ny); - f_merge_result.set_size(nx, ny); - o_merge_result.set_size(nx, ny); - - combine_boolplanes("Fcst_Simple", f_simple_plane, n_fcst_files, f_calc, f_simple_result); - combine_boolplanes("Obs_Simple", o_simple_plane, n_obs_files, o_calc, o_simple_result); - combine_boolplanes("Fcst_Merge", f_merge_plane, n_fcst_files, f_calc, f_merge_result); - combine_boolplanes("Obs_Merge", o_merge_plane, n_obs_files, o_calc, o_merge_result); - - - // create ShapeData objects using something from mvd as a template - // (shape data has 1's or bad) - - f_simple_sd = ShapeData(*(mvdFcst[0]->_simple->_sd)); - for (int x=0; x_simple->_sd)); - for (int x=0; x_simple->_sd)); - for (int x=0; x_simple->_sd)); - for (int x=0; x_sd->data); - mask_data("Obs", nx, ny, o_result, mvdo._simple->_sd->data); - - // - // build the command for running mode frontend - // - StringArray mode_argv; - char junk [256]; - - mode_argv.clear(); - mode_argv.add(mode_path); - mode_argv.add(fcst_filename); - mode_argv.add(obs_filename); - mode_argv.add(config_file); - mode_argv.add("-v"); - snprintf(junk, sizeof(junk), "%d", mlog.verbosity_level()); - mode_argv.add(junk); - mode_argv.add("-outdir"); - mode_argv.add(dir); + mask_data("Fcst", nx, ny, fsuper._simple_result, mvdf._simple->_sd->data); + mask_data("Obs", nx, ny, osuper._simple_result, mvdo._simple->_sd->data); mlog << Debug(1) << "Running mvmode intensity comparisions \n\n"; ModeFrontEnd *frontend = new ModeFrontEnd; - int status = frontend->multivar_intensity_comparisons(mode_argv, mvdf, mvdo, has_union_f, - has_union_o, merge_f, merge_o, findex, - oindex); + int status = frontend->multivar_intensity_comparisons(config, mvdf, mvdo, has_union_f, + has_union_o, + fsuper._merge_sd_split, + osuper._merge_sd_split, + findex, + oindex, + fcst_filename, + obs_filename, + config_file, dir); delete frontend; } //////////////////////////////////////////////////////////////////////// -void process_superobjects(ShapeData &f_result, ShapeData &o_result, - ShapeData &f_merge, ShapeData &o_merge, +void process_superobjects(ModeSuperObject &fsuper, + ModeSuperObject &osuper, int nx, int ny, const ConcatString &dir, - GrdFileType ftype, GrdFileType otype, const Grid &grid, - bool has_union) + GrdFileType ftype, GrdFileType otype, + const Grid &grid, bool has_union) { - StringArray mode_argv; - char junk [256]; - - // - // build the command for running mode frontend - // - mode_argv.clear(); - mode_argv.add(mode_path); - mode_argv.add(config_file); - mode_argv.add("-v"); - snprintf(junk, sizeof(junk), "%d", mlog.verbosity_level()); - mode_argv.add(junk); - mode_argv.add("-outdir"); - mode_argv.add(dir); - mlog << Debug(1) << "Running superobject mode \n\n"; // set the data to 0 inside superobjects and missing everywhere else - mask_data_super("FcstSimple", nx, ny, f_result.data); - mask_data_super("ObsSimple", nx, ny, o_result.data); + mask_data_super("FcstSimple", nx, ny, fsuper._simple_sd.data); + mask_data_super("ObsSimple", nx, ny, osuper._simple_sd.data); ModeFrontEnd *frontend = new ModeFrontEnd; - int status = frontend->run_super(mode_argv, f_result, o_result, - f_merge, o_merge, ftype, otype, grid, has_union); + + int status = frontend->run_super(config, fsuper, osuper, + ftype, otype, grid, has_union, + config_file, dir); delete frontend; } @@ -883,31 +778,3 @@ int _mkdir(const char *dir) return (mkdir(tmp, dir_creation_mode)); } - -//////////////////////////////////////////////////////////////////////// - -void _debug_shape_examine(string &name, const ShapeData &sd, int nx, int ny) -{ - vector values; - vector count; - for (int x=0; x::iterator vi; - vi = find(values.begin(), values.end(), v); - if (vi == values.end()) { - values.push_back(v); - count.push_back(1); - } else { - int ii = vi - values.begin(); - count[ii] = count[ii] + 1; - } - } - } - for (size_t i=0; i Date: Thu, 28 Dec 2023 22:27:33 +0000 Subject: [PATCH 02/14] separated multivar and traditional mode frontend classes to simplify the mode executive class design --- src/tools/core/mode/mode.cc | 11 +- src/tools/core/mode/mode_exec.cc | 410 +++++++----- src/tools/core/mode/mode_exec.h | 51 +- src/tools/core/mode/mode_frontend.cc | 504 +-------------- src/tools/core/mode/mode_frontend.h | 64 +- src/tools/core/mode/mode_superobject.cc | 82 +++ src/tools/core/mode/mode_superobject.h | 6 + src/tools/core/mode/multivar_frontend.cc | 777 +++++++++++------------ src/tools/core/mode/multivar_frontend.h | 106 ++++ 9 files changed, 891 insertions(+), 1120 deletions(-) create mode 100644 src/tools/core/mode/multivar_frontend.h diff --git a/src/tools/core/mode/mode.cc b/src/tools/core/mode/mode.cc index 68463a6c84..d71a7698b5 100644 --- a/src/tools/core/mode/mode.cc +++ b/src/tools/core/mode/mode.cc @@ -57,6 +57,7 @@ // 020 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main // 021 06/09/23 Albo Major changes for multivariate mode // 022 11/02/23 Halley Gotway MET #2724 add OpenMP to convolution +// 023 12/27/23 Albo MET #2745 more unit tests, read data one time, percentile thresholding // //////////////////////////////////////////////////////////////////////// @@ -78,6 +79,7 @@ using namespace std; #include "string_array.h" #include "mode_usage.h" #include "mode_frontend.h" +#include "multivar_frontend.h" #include "mode_conf_info.h" #ifdef WITH_PYTHON @@ -97,7 +99,6 @@ using namespace std; /////////////////////////////////////////////////////////////////////// -extern int mode_frontend(const StringArray &); extern int multivar_frontend(const StringArray &); extern const char * const program_name; @@ -186,13 +187,15 @@ int met_main(int argc, char * argv []) config.check_multivar_not_implemented(); // run the multivar version of mode - - status = multivar_frontend(Argv); + + MultivarFrontEnd *frontend = new MultivarFrontEnd(); + status = frontend->run(Argv); + if ( frontend ) { delete frontend; frontend = 0; } } else { - // run the traditional version of mode, with command line arguments as is + // run the traditional version of mode ModeFrontEnd *frontend = new ModeFrontEnd; status = frontend->run_traditional(Argv); diff --git a/src/tools/core/mode/mode_exec.cc b/src/tools/core/mode/mode_exec.cc index 28818ba8b8..2f44e944c0 100644 --- a/src/tools/core/mode/mode_exec.cc +++ b/src/tools/core/mode/mode_exec.cc @@ -60,12 +60,11 @@ static void replaceAll(std::string& str, const std::string& from, const std::str /////////////////////////////////////////////////////////////////////// -ModeExecutive::ModeExecutive(Processing_t p) +ModeExecutive::ModeExecutive()//Processing_t p) { init_from_scratch(); - ptype = p; } @@ -132,8 +131,6 @@ void ModeExecutive::clear() R_index = T_index = 0; - ptype = TRADITIONAL; - // // done // @@ -206,24 +203,6 @@ void ModeExecutive::init_traditional(int n_files) /////////////////////////////////////////////////////////////////////// - -void ModeExecutive::init_multivar_verif_grid(const DataPlane &fcst, - const DataPlane &obs, - const ModeConfInfo &conf) -{ - - R_index = T_index = 0; - engine.conf_info = conf; - engine.conf_info.check_multivar_not_implemented(); - - // engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); - - return; - -} - -/////////////////////////////////////////////////////////////////////// - void ModeExecutive::init_multivar_simple(int j, int n_files, ModeDataType dtype, const ModeConfInfo &conf) @@ -268,7 +247,7 @@ void ModeExecutive::init_multivar_intensities(GrdFileType ftype, GrdFileType oty /////////////////////////////////////////////////////////////////////// -void ModeExecutive::setup_fcst_obs_data_traditional() +void ModeExecutive::setup_traditional_fcst_obs_data() { @@ -282,7 +261,7 @@ void ModeExecutive::setup_fcst_obs_data_traditional() if ( !(fcst_mtddf->data_plane(*(engine.conf_info.Fcst->var_info), Fcst_sd.data)) ) { - mlog << Error << "\nModeExecutive::setup_fcst_obs_data_traditional() -> " + mlog << Error << "\nModeExecutive::setup_traditionalfcst_obs_data() -> " << "can't get forecast data \"" << engine.conf_info.Fcst->var_info->magic_str() << "\" from file \"" << fcst_file << "\"\n\n"; @@ -293,7 +272,7 @@ void ModeExecutive::setup_fcst_obs_data_traditional() if ( !(obs_mtddf->data_plane(*(engine.conf_info.Obs->var_info), Obs_sd.data)) ) { - mlog << Error << "\nModeExecutive::setup_fcst_obs_data_traditional() -> " + mlog << Error << "\nModeExecutive::setup_traditional_fcst_obs_data() -> " << "can't get observation data \"" << engine.conf_info.Obs->var_info->magic_str() << "\" from file \"" << obs_file << "\"\n\n"; @@ -341,7 +320,7 @@ void ModeExecutive::setup_fcst_obs_data_traditional() if(Fcst_sd.data.valid() != Obs_sd.data.valid()) { - mlog << Warning << "\nModeExecutive::setup_fcst_obs_data_traditional() -> " + mlog << Warning << "\nModeExecutive::setup_traditional_fcst_obs_data() -> " << "Forecast and observation valid times do not match " << unix_to_yyyymmdd_hhmmss(Fcst_sd.data.valid()) << " != " << unix_to_yyyymmdd_hhmmss(Obs_sd.data.valid()) << " for " @@ -355,7 +334,7 @@ void ModeExecutive::setup_fcst_obs_data_traditional() engine.conf_info.Obs->var_info->level().type() == LevelType_Accum && Fcst_sd.data.accum() != Obs_sd.data.accum()) { - mlog << Warning << "\nModeExecutive::setup_fcst_obs_data_traditional() -> " + mlog << Warning << "\nModeExecutive::setup_traditional_fcst_obs_data() -> " << "Forecast and observation accumulation times do not match " << sec_to_hhmmss(Fcst_sd.data.valid()) << " != " << sec_to_hhmmss(Obs_sd.data.valid()) << " for " @@ -423,22 +402,19 @@ void ModeExecutive::setup_fcst_obs_data_traditional() /////////////////////////////////////////////////////////////////////// - void ModeExecutive::setup_verification_grid(const ModeInputData &fcst, - const ModeInputData &obs) + const ModeInputData &obs, + const ModeConfInfo &conf) { - - // ShapeData fcst_sd, obs_sd; - double fmin, omin, fmax, omax; + R_index = T_index = 0; + engine.conf_info = conf; + engine.conf_info.check_multivar_not_implemented(); Fcst_sd.clear(); Obs_sd.clear(); - Fcst_sd.data = fcst._dataPlane; Obs_sd.data = obs._dataPlane; - // Determine the verification grid - engine.conf_info.set_field_index(0); grid = parse_vx_grid(engine.conf_info.Fcst->var_info->regrid(), &fcst._grid, &obs._grid); @@ -447,9 +423,8 @@ void ModeExecutive::setup_verification_grid(const ModeInputData &fcst, /////////////////////////////////////////////////////////////////////// - -void ModeExecutive::setup_fcst_data(const Grid &verification_grid, - const ModeInputData &input) +void ModeExecutive::setup_multivar_fcst_data(const Grid &verification_grid, + const ModeInputData &input) { double fmin, fmax; @@ -511,9 +486,8 @@ void ModeExecutive::setup_fcst_data(const Grid &verification_grid, /////////////////////////////////////////////////////////////////////// - -void ModeExecutive::setup_obs_data(const Grid &verification_grid, - const ModeInputData &input) +void ModeExecutive::setup_multivar_obs_data(const Grid &verification_grid, + const ModeInputData &input) { // ShapeData fcst_sd, obs_sd; @@ -579,7 +553,7 @@ void ModeExecutive::setup_obs_data(const Grid &verification_grid, /////////////////////////////////////////////////////////////////////// -void ModeExecutive::setup_fcst_obs_data_multivar_intensities(const MultiVarData &mvdf, +void ModeExecutive::setup_multivar_fcst_obs_data_intensities(const MultiVarData &mvdf, const MultiVarData &mvdo) { double fmin, fmax, omin, omax; @@ -600,7 +574,7 @@ void ModeExecutive::setup_fcst_obs_data_multivar_intensities(const MultiVarData // Print a warning if the valid times do not match if(Fcst_sd.data.valid() != Obs_sd.data.valid()) { - mlog << Warning << "\nModeExecutive::setup_fcst_obs_data_multivar_intensities() -> " + mlog << Warning << "\nModeExecutive::setup_multivar_fcst_obs_data_intensities() -> " << "Forecast and observation valid times do not match " << unix_to_yyyymmdd_hhmmss(Fcst_sd.data.valid()) << " != " << unix_to_yyyymmdd_hhmmss(Obs_sd.data.valid()) << " for " @@ -613,7 +587,7 @@ void ModeExecutive::setup_fcst_obs_data_multivar_intensities(const MultiVarData engine.conf_info.Obs->var_info->level().type() == LevelType_Accum && Fcst_sd.data.accum() != Obs_sd.data.accum()) { - mlog << Warning << "\nModeExecutive::setup_fcst_obs_data_multivar_intensities() -> " + mlog << Warning << "\nModeExecutive::setup_multivar_fcst_obs_data_intensities() -> " << "Forecast and observation accumulation times do not match " << sec_to_hhmmss(Fcst_sd.data.valid()) << " != " << sec_to_hhmmss(Obs_sd.data.valid()) << " for " @@ -669,7 +643,7 @@ void ModeExecutive::setup_fcst_obs_data_multivar_intensities(const MultiVarData /////////////////////////////////////////////////////////////////////// -void ModeExecutive::setup_fcst_obs_data_multivar_super(const ShapeData &f_super, +void ModeExecutive::setup_multivar_fcst_obs_data_super(const ShapeData &f_super, const ShapeData &o_super, const Grid &igrid) { @@ -697,7 +671,7 @@ void ModeExecutive::setup_fcst_obs_data_multivar_super(const ShapeData &f_super, // Print a warning if the valid times do not match if(Fcst_sd.data.valid() != Obs_sd.data.valid()) { - mlog << Warning << "\nModeExecutive::setup_fcst_obs_data_multivar_super() -> " + mlog << Warning << "\nModeExecutive::setup_multivar_fcst_obs_data_super() -> " << "Forecast and observation valid times do not match " << unix_to_yyyymmdd_hhmmss(Fcst_sd.data.valid()) << " != " << unix_to_yyyymmdd_hhmmss(Obs_sd.data.valid()) << " for " @@ -710,7 +684,7 @@ void ModeExecutive::setup_fcst_obs_data_multivar_super(const ShapeData &f_super, engine.conf_info.Obs->var_info->level().type() == LevelType_Accum && Fcst_sd.data.accum() != Obs_sd.data.accum()) { - mlog << Warning << "\nModeExecutive::setup_fcst_obs_data_multivar_super() -> " + mlog << Warning << "\nModeExecutive::setup_multivar_fcst_obs_data_super() -> " << "Forecast and observation accumulation times do not match " << sec_to_hhmmss(Fcst_sd.data.valid()) << " != " << sec_to_hhmmss(Obs_sd.data.valid()) << " for " @@ -756,7 +730,7 @@ void ModeExecutive::setup_fcst_obs_data_multivar_super(const ShapeData &f_super, /////////////////////////////////////////////////////////////////////// -void ModeExecutive::do_conv_thresh(const int r_index, const int t_index) +void ModeExecutive::do_conv_thresh_traditional(const int r_index, const int t_index) { ModeConfInfo & conf = engine.conf_info; @@ -768,65 +742,162 @@ void ModeExecutive::do_conv_thresh(const int r_index, const int t_index) R_index = r_index; T_index = t_index; - if (ptype == MULTIVAR_SIMPLE_MERGE) { - conf.set_conv_thresh_by_merge_index(T_index); - } else if (ptype == MULTIVAR_SUPER) { - SingleThresh s("ne-9999"); - conf.set_conv_thresh(s); - conf.set_conv_radius(0.0); + conf.set_conv_radius_by_index(R_index); + conf.set_conv_thresh_by_index(T_index); + conf.set_merge_thresh_by_index(T_index); + + // + // Set up the engine with these raw fields + // + + string what = "forecast and observation fields"; + mlog << Debug(2) << "Identifying objects in the " << what << "...\n"; + + if ( r_index != local_r_index ) { + // need to do convolution + engine.set(Fcst_sd, Obs_sd); } else { - // this could break down if obs and fcst have different arrays - // currently not allowed in multivar mode, should work correctly - // for traditional mode - conf.set_conv_radius_by_index(R_index); - conf.set_conv_thresh_by_index(T_index); + // don't need to do convolution + engine.set_no_conv(Fcst_sd, Obs_sd); } + // + // Compute the contingency table statistics for the fields, if needed + // + if ( conf.ct_stats_flag ) compute_ct_stats(); + + // + // done + // + + local_r_index = r_index; + + return; + +} + +/////////////////////////////////////////////////////////////////////// + +void ModeExecutive::do_conv_thresh_multivar_super() +{ + + ModeConfInfo & conf = engine.conf_info; + + R_index = 0; + T_index = 0; + + SingleThresh s("ne-9999"); + conf.set_conv_thresh(s); + conf.set_conv_radius(0.0); conf.set_merge_thresh_by_index(T_index); // // Set up the engine with these raw fields // - string what = "forecast and observation fields"; - if (ptype == MULTIVAR_SIMPLE || ptype == MULTIVAR_SIMPLE_MERGE) { - if (conf.data_type == ModeDataType_MvMode_Obs) { - what = "observation field"; - } else { - what = "forecast field"; - } - } mlog << Debug(2) << "Identifying objects in the " << what << "...\n"; - if ( r_index != local_r_index ) { // need to do convolution + engine.set_only_split(Fcst_sd, Obs_sd); - if (ptype == MULTIVAR_INTENSITY || ptype == MULTIVAR_SUPER) { - engine.set_only_split(Fcst_sd, Obs_sd); - } else {// MULTIVAR_SIMPLE, MULTIVAR_SIMPLE_MERGE, TRADITIONAL - engine.set(Fcst_sd, Obs_sd); - } - } else { // don't need to do convolution + // + // Compute the contingency table statistics for the fields, if needed + // (not needed for simple or merge, only one field) + // + if ( conf.ct_stats_flag ) compute_ct_stats(); - if (ptype == MULTIVAR_INTENSITY || ptype == MULTIVAR_SUPER) { - engine.set_only_split(Fcst_sd, Obs_sd); - } else {// MULTIVAR_SIMPLE, MULTIVAR_SIMPLE_MERGE, TRADITIONAL - engine.set_no_conv(Fcst_sd, Obs_sd); - } - } + // + // done + // + + local_r_index = 0; + + return; + +} + + +/////////////////////////////////////////////////////////////////////// + + +void ModeExecutive::do_conv_thresh_multivar_intensity_compare() +{ + + ModeConfInfo & conf = engine.conf_info; + + R_index = 0; + T_index = 0; + + conf.set_conv_radius_by_index(R_index); + conf.set_conv_thresh_by_index(T_index); + conf.set_merge_thresh_by_index(T_index); + + // + // Set up the engine with these raw fields + // + + string what = "forecast and observation fields"; + mlog << Debug(2) << "Identifying objects in the " << what << "...\n"; + + engine.set_only_split(Fcst_sd, Obs_sd); // // Compute the contingency table statistics for the fields, if needed - // (not needed for simple or merge, only one field) // - if (ptype != MULTIVAR_SIMPLE && ptype != MULTIVAR_SIMPLE_MERGE) { - if ( conf.ct_stats_flag ) compute_ct_stats(); - } + if ( conf.ct_stats_flag ) compute_ct_stats(); // // done // - local_r_index = r_index; + local_r_index = 0; + + return; + +} + +/////////////////////////////////////////////////////////////////////// + +void ModeExecutive::do_conv_thresh_multivar_simple(Processing_t p) +{ + + ModeConfInfo & conf = engine.conf_info; + + R_index = 0; + T_index = 0; + + if (p == MULTIVAR_SIMPLE_MERGE) { + conf.set_conv_thresh_by_merge_index(T_index); + } else if (p == MULTIVAR_SIMPLE) { + conf.set_conv_radius_by_index(0); + conf.set_conv_thresh_by_index(0); + } else { + mlog << Error << "\nModeExecutive::do_conv_thresh_multivar_simple() -> " + << "Wrong processing type input " << stype(p) << "\"\n\n"; + exit(1); + } + + conf.set_merge_thresh_by_index(0); + + // + // Set up the engine with these raw fields + // + + string what; + if (conf.data_type == ModeDataType_MvMode_Obs) { + what = "observation field"; + } else { + what = "forecast field"; + } + mlog << Debug(2) << "Identifying objects in the " << what << "...\n"; + + engine.set(Fcst_sd, Obs_sd); + + // (ct_stats not needed for simple or merge, only one field) + // + // done + // + + local_r_index = 0; return; @@ -841,7 +912,7 @@ void ModeExecutive::clear_internal_r_index() /////////////////////////////////////////////////////////////////////// -void ModeExecutive::do_merging() +void ModeExecutive::do_merging_traditional() { mlog << Debug(2) << "Identified: " << engine.n_fcst << " forecast objects " @@ -873,10 +944,11 @@ void ModeExecutive::do_merging() /////////////////////////////////////////////////////////////////////// -void ModeExecutive::do_merging(const ShapeData &f_merge, const ShapeData &o_merge) - +void ModeExecutive::do_merging_multivar(const ShapeData &f_merge, + const ShapeData &o_merge, + Processing_t p) { - if (ptype == MULTIVAR_SUPER) { + if (p == MULTIVAR_SUPER) { // set the merge flag and merge_thresh appropriately ModeConfInfo & conf = engine.conf_info; SingleThresh s("ne-9999"); @@ -884,14 +956,11 @@ void ModeExecutive::do_merging(const ShapeData &f_merge, const ShapeData &o_merg conf.set_fcst_merge_thresh(s); conf.set_obs_merge_flag(MergeType_Thresh); conf.set_obs_merge_thresh(s); - } else { - if (ptype != MULTIVAR_INTENSITY) { - mlog << Error << "\nModeExecutive::do_merging(shapedata, shapedata) -> " - << "wrong method for processing type " << stype(ptype) << "\n\n"; - exit(1); - } + } else if (p != MULTIVAR_INTENSITY) { + mlog << Error << "\nModeExecutive::do_merging(shapedata, shapedata, p) -> " + << "wrong method for processing type " << stype(p) << "\n\n"; + exit(1); } - mlog << Debug(2) << "Identified: " << engine.n_fcst << " forecast objects " @@ -926,11 +995,12 @@ void ModeExecutive::do_merging(const ShapeData &f_merge, const ShapeData &o_merg /////////////////////////////////////////////////////////////////////// -void ModeExecutive::do_match_merge(const ShapeData &f_merge, - const ShapeData &o_merge) +void ModeExecutive::do_match_merge_multivar(const ShapeData &f_merge, + const ShapeData &o_merge, + Processing_t p) { - do_merging(f_merge, o_merge); + do_merging_multivar(f_merge, o_merge, p); // Do the matching of objects between fields @@ -943,10 +1013,10 @@ void ModeExecutive::do_match_merge(const ShapeData &f_merge, /////////////////////////////////////////////////////////////////////// -void ModeExecutive::do_match_merge() +void ModeExecutive::do_match_merge_traditional() { - do_merging(); + do_merging_traditional(); // Do the matching of objects between fields @@ -1117,82 +1187,104 @@ void ModeExecutive::set_raw_to_full(float *fcst_raw_data, /////////////////////////////////////////////////////////////////////// -void ModeExecutive::process_output(const MultiVarData *mvdf, - const MultiVarData *mvdo) - +void ModeExecutive::process_output_multivar_super() { + isMultivarOutput = false; + isMultivarSuperOutput = true; - // store to class member so don't have to pass it around - isMultivarOutput = (ptype == MULTIVAR_INTENSITY); - isMultivarSuperOutput = (ptype == MULTIVAR_SUPER); + // use the configured multivar name and level + fcst_magic_string = engine.conf_info.fcst_multivar_name.string() + "_" + engine.conf_info.fcst_multivar_level.string(); + obs_magic_string = engine.conf_info.obs_multivar_name.string() + "_" + engine.conf_info.obs_multivar_level.string(); - if (isMultivarOutput) { + // Create output stats files and plots - // get the magic strings, which will be used in file naming - fcst_magic_string = engine.conf_info.Fcst->var_info->magic_str().c_str(); - obs_magic_string = engine.conf_info.Obs->var_info->magic_str().c_str(); + write_obj_stats(); - // replace forward slashes with underscores to prevent new directories - replace(fcst_magic_string.begin(), fcst_magic_string.end(), '/', '_'); - replace(obs_magic_string.begin(), obs_magic_string.end(), '/', '_'); + if ( engine.conf_info.ct_stats_flag ) write_ct_stats(); + + write_obj_netcdf(engine.conf_info.nc_info); - // replace (*,*) with '_all_all_' - replaceAll(fcst_magic_string, "*", "all"); - replaceAll(obs_magic_string, "*", "all"); - replaceAll(fcst_magic_string, ",", "_"); - replaceAll(obs_magic_string, ",", "_"); - replaceAll(fcst_magic_string, "(", "_"); - replaceAll(obs_magic_string, "(", "_"); - replaceAll(fcst_magic_string, ")", ""); - replaceAll(obs_magic_string, ")", ""); - - } else if (isMultivarSuperOutput) { + if ( engine.conf_info.ps_plot_flag ) plot_engine(); - // use the configured multivar name and level - fcst_magic_string = engine.conf_info.fcst_multivar_name.string() + "_" + engine.conf_info.fcst_multivar_level.string(); - obs_magic_string = engine.conf_info.obs_multivar_name.string() + "_" + engine.conf_info.obs_multivar_level.string(); + return; - } else { +} - // just in case make these empty - fcst_magic_string = ""; - obs_magic_string = ""; +/////////////////////////////////////////////////////////////////////// - } - +void ModeExecutive::process_output_multivar_intensity_compare(const MultiVarData *mvdf, + const MultiVarData *mvdo) + +{ + isMultivarOutput = true; + isMultivarSuperOutput = false; + + // get the magic strings, which will be used in file naming + fcst_magic_string = engine.conf_info.Fcst->var_info->magic_str().c_str(); + obs_magic_string = engine.conf_info.Obs->var_info->magic_str().c_str(); + + // replace forward slashes with underscores to prevent new directories + replace(fcst_magic_string.begin(), fcst_magic_string.end(), '/', '_'); + replace(obs_magic_string.begin(), obs_magic_string.end(), '/', '_'); + + // replace (*,*) with '_all_all_' + replaceAll(fcst_magic_string, "*", "all"); + replaceAll(obs_magic_string, "*", "all"); + replaceAll(fcst_magic_string, ",", "_"); + replaceAll(obs_magic_string, ",", "_"); + replaceAll(fcst_magic_string, "(", "_"); + replaceAll(obs_magic_string, "(", "_"); + replaceAll(fcst_magic_string, ")", ""); + replaceAll(obs_magic_string, ")", ""); + // Create output stats files and plots write_obj_stats(); if ( engine.conf_info.ct_stats_flag ) write_ct_stats(); - - if (isMultivarOutput) { - if (mvdf && mvdo) { - double fmin = mvdf->_data_min; - double fmax = mvdf->_data_max; - double omin = mvdo->_data_min; - double omax = mvdo->_data_max; - double data_min, data_max; - if (!is_bad_data(fmin) && !is_bad_data(omin)) data_min = min(fmin, omin); - else if(!is_bad_data(fmin) && is_bad_data(omin)) data_min = fmin; - else if( is_bad_data(fmin) && !is_bad_data(omin)) data_min = omin; - - if (!is_bad_data(fmax) && !is_bad_data(omax)) data_max = max(fmax, omax); - else if(!is_bad_data(fmax) && is_bad_data(omax)) data_max = fmax; - else if( is_bad_data(fmax) && !is_bad_data(omax)) data_max = omax; + double fmin = mvdf->_data_min; + double fmax = mvdf->_data_max; + double omin = mvdo->_data_min; + double omax = mvdo->_data_max; + double data_min, data_max; + if (!is_bad_data(fmin) && !is_bad_data(omin)) data_min = min(fmin, omin); + else if(!is_bad_data(fmin) && is_bad_data(omin)) data_min = fmin; + else if( is_bad_data(fmin) && !is_bad_data(omin)) data_min = omin; + + if (!is_bad_data(fmax) && !is_bad_data(omax)) data_max = max(fmax, omax); + else if(!is_bad_data(fmax) && is_bad_data(omax)) data_max = fmax; + else if( is_bad_data(fmax) && !is_bad_data(omax)) data_max = omax; - set_raw_to_full(mvdf->_simple->_raw_data, - mvdo->_simple->_raw_data, - mvdf->_nx, mvdf->_ny, - data_min, data_max); - } else { - mlog << Error << "\nModeExecutive::process_output() -> " - << "no multivar data when multivar data is expected\n\n"; - exit(1); - } - } + set_raw_to_full(mvdf->_simple->_raw_data,mvdo->_simple->_raw_data, + mvdf->_nx, mvdf->_ny, data_min, data_max); + + write_obj_netcdf(engine.conf_info.nc_info); + + if ( engine.conf_info.ps_plot_flag ) plot_engine(); + + return; + +} + +/////////////////////////////////////////////////////////////////////// + +void ModeExecutive::process_output_traditional() +{ + isMultivarOutput = false; + isMultivarSuperOutput = false; + + // just in case make these empty + fcst_magic_string = ""; + obs_magic_string = ""; + + // Create output stats files and plots + + write_obj_stats(); + + if ( engine.conf_info.ct_stats_flag ) write_ct_stats(); + write_obj_netcdf(engine.conf_info.nc_info); if ( engine.conf_info.ps_plot_flag ) plot_engine(); diff --git a/src/tools/core/mode/mode_exec.h b/src/tools/core/mode/mode_exec.h index c326c6ab30..574bf937c6 100644 --- a/src/tools/core/mode/mode_exec.h +++ b/src/tools/core/mode/mode_exec.h @@ -75,14 +75,12 @@ class ModeExecutive { // the various mode algorithm settings typedef enum {TRADITIONAL, MULTIVAR_SIMPLE, MULTIVAR_SIMPLE_MERGE, MULTIVAR_INTENSITY, MULTIVAR_SUPER} Processing_t; - ModeExecutive(Processing_t p=TRADITIONAL); + ModeExecutive(); ~ModeExecutive(); void clear(); void init_traditional(int n_files); - void init_multivar_verif_grid(const DataPlane &fcst, - const DataPlane &obs, const ModeConfInfo &config); void init_multivar_simple(int j, int n_files, ModeDataType dtype, const ModeConfInfo &conf); void init_multivar_intensities(GrdFileType ftype, GrdFileType otype, const ModeConfInfo &conf); @@ -132,29 +130,44 @@ class ModeExecutive { bool isMultivarOutput; bool isMultivarSuperOutput; - Processing_t ptype; + //Processing_t ptype; - void clear_internal_r_index(); void setup_verification_grid(const ModeInputData &fcst, - const ModeInputData &obs); - void setup_fcst_obs_data_traditional(); - void setup_fcst_data(const Grid &verification_grid, const ModeInputData &input); - void setup_obs_data(const Grid &verification_grid, const ModeInputData &input); - void setup_fcst_obs_data_multivar_intensities(const MultiVarData &mvdf, + const ModeInputData &obs, + const ModeConfInfo &conf); + + void clear_internal_r_index(); + + void setup_traditional_fcst_obs_data(); + void setup_multivar_fcst_data(const Grid &verification_grid, const ModeInputData &input); + void setup_multivar_obs_data(const Grid &verification_grid, const ModeInputData &input); + void setup_multivar_fcst_obs_data_intensities(const MultiVarData &mvdf, const MultiVarData &mvdo); - void setup_fcst_obs_data_multivar_super(const ShapeData &f_super, - const ShapeData &o_super, const Grid &igrid); - void do_conv_thresh(const int r_index, const int t_index); - void do_merging(); - void do_merging(const ShapeData &f_merge, const ShapeData &o_merge); - void do_match_merge(); - void do_match_merge(const ShapeData &f_merge, const ShapeData &o_merge); + void setup_multivar_fcst_obs_data_super(const ShapeData &f_super, + const ShapeData &o_super, + const Grid &igrid); + + void do_conv_thresh_traditional(const int r_index, const int t_index); + void do_conv_thresh_multivar_super(); + void do_conv_thresh_multivar_intensity_compare(); + void do_conv_thresh_multivar_simple(Processing_t p); + + void do_merging_traditional(); + void do_merging_multivar(const ShapeData &f_merge, const ShapeData &o_merge, + Processing_t p); + + void do_match_merge_traditional(); + void do_match_merge_multivar(const ShapeData &f_merge, const ShapeData &o_merge, + Processing_t p); void process_masks(ShapeData &, ShapeData &); void process_fcst_masks(ShapeData &); void process_obs_masks(ShapeData &); - void process_output(const MultiVarData *mvdf=NULL, - const MultiVarData *mvdo=NULL); + + void process_output_traditional(); + void process_output_multivar_intensity_compare(const MultiVarData *mvdf, + const MultiVarData *mvdo); + void process_output_multivar_super(); void set_raw_to_full(float *fcst_raw_data, float *obs_raw_data, diff --git a/src/tools/core/mode/mode_frontend.cc b/src/tools/core/mode/mode_frontend.cc index efbe774485..4831cb59d2 100644 --- a/src/tools/core/mode/mode_frontend.cc +++ b/src/tools/core/mode/mode_frontend.cc @@ -35,13 +35,11 @@ using namespace std; extern const char * const program_name; static ModeExecutive *mode_exec = 0; -static ModeExecutive::Processing_t ptype = ModeExecutive::TRADITIONAL; // used only for traditional mode, multivar sets it into config previous // to the frontend creation static int compress_level = -1; - /////////////////////////////////////////////////////////////////////// @@ -64,154 +62,13 @@ ModeFrontEnd::~ModeFrontEnd() } -/////////////////////////////////////////////////////////////////////// - -Grid ModeFrontEnd::create_verification_grid(const ModeInputData &fcst, - const ModeInputData &obs, - const string &config_file, - const ModeConfInfo &config) -{ - if ( mode_exec ) { delete mode_exec; mode_exec = 0; } - mode_exec = new ModeExecutive; - mode_exec->out_dir = default_out_dir; - mode_exec->fcst_file = "None"; - mode_exec->obs_file = "None"; - mode_exec->match_config_file = config_file; - mode_exec->init_multivar_verif_grid(fcst._dataPlane, obs._dataPlane, config); - - ModeConfInfo & conf = mode_exec->engine.conf_info; - conf.set_field_index(0); - mode_exec->setup_verification_grid(fcst, obs); - Grid g = mode_exec->grid; - delete mode_exec; mode_exec = 0; - return g; -} - - -/////////////////////////////////////////////////////////////////////// - -int -ModeFrontEnd::create_multivar_simple_objects( - const ModeConfInfo &config, - ModeDataType dtype, - const Grid &verification_grid, - const ModeInputData &input, - const string &filename, - const string &config_file, - const string &outdir, - int field_index, int n_files) - -{ - init(ModeExecutive::MULTIVAR_SIMPLE); - - mode_exec->out_dir = replace_path(default_out_dir); - if (dtype == ModeDataType_MvMode_Fcst) { - mode_exec->fcst_file = filename; - mode_exec->obs_file = "None"; - } else { - mode_exec->obs_file = filename; - mode_exec->fcst_file = "None"; - } - mode_exec->match_config_file = config_file; - mode_exec->out_dir = outdir; - mode_exec->init_multivar_simple(field_index, n_files, dtype, config); - - ModeConfInfo & conf = mode_exec->engine.conf_info; - - if (dtype == ModeDataType_MvMode_Fcst) { - mode_exec->setup_fcst_data(verification_grid, input); - } else { - mode_exec->setup_obs_data(verification_grid, input); - } - - // - // mode algorithm - // - if ( conf.quilt ) { - - do_quilt(); - - } else { - - do_straight(); - - } - - // - // done - // - -#ifdef WITH_PYTHON - GP.finalize(); -#endif - return (0); -} - -/////////////////////////////////////////////////////////////////////// - -int -ModeFrontEnd::create_multivar_merge_objects(const ModeConfInfo &config, - ModeDataType dtype, - const Grid &verification_grid, - const ModeInputData &input, - const string &filename, - const string &config_file, - const string &outdir, int field_index, - int n_files) -{ - init(ModeExecutive::MULTIVAR_SIMPLE_MERGE); - - mode_exec->out_dir = replace_path(default_out_dir); - if (dtype == ModeDataType_MvMode_Fcst) { - mode_exec->fcst_file = filename; - mode_exec->obs_file = "None"; - } else { - mode_exec->obs_file = filename; - mode_exec->fcst_file = "None"; - } - mode_exec->match_config_file = config_file; - mode_exec->out_dir = outdir; - - mode_exec->init_multivar_simple(field_index, n_files, dtype, config); - - ModeConfInfo & conf = mode_exec->engine.conf_info; - - if (dtype == ModeDataType_MvMode_Fcst) { - mode_exec->setup_fcst_data(verification_grid, input); - } else { - mode_exec->setup_obs_data(verification_grid, input); - } - - - // - // mode algorithm - // - if ( conf.quilt ) { - - do_quilt(); - - } else { - - do_straight(); - - } - - // - // done - // - -#ifdef WITH_PYTHON - GP.finalize(); -#endif - return (0); -} /////////////////////////////////////////////////////////////////////// int ModeFrontEnd::run_traditional(const StringArray & Argv) { - init(ModeExecutive::TRADITIONAL); + init(); int field_index = -1; int n_files = 1; @@ -220,7 +77,7 @@ int ModeFrontEnd::run_traditional(const StringArray & Argv) // Process the command line arguments // - process_command_line(Argv, false); + process_command_line(Argv); mode_exec->init_traditional(n_files); @@ -233,7 +90,7 @@ int ModeFrontEnd::run_traditional(const StringArray & Argv) // read in data // - mode_exec->setup_fcst_obs_data_traditional(); + mode_exec->setup_traditional_fcst_obs_data(); // // mode algorithm @@ -260,133 +117,6 @@ int ModeFrontEnd::run_traditional(const StringArray & Argv) /////////////////////////////////////////////////////////////////////// -int -ModeFrontEnd::multivar_intensity_comparisons(const ModeConfInfo &config, - const MultiVarData &mvdf, - const MultiVarData &mvdo, bool has_union_f, - bool has_union_o, const ShapeData &merge_f, - const ShapeData &merge_o, - int field_index_f, int field_index_o, - const string &fcst_filename, - const string &obs_filename, - const string &config_file, const string &dir) -{ - init(ModeExecutive::MULTIVAR_INTENSITY); - - mode_exec->out_dir = replace_path(default_out_dir); - mode_exec->fcst_file = fcst_filename; - mode_exec->obs_file = obs_filename; - mode_exec->match_config_file = config_file; - mode_exec->out_dir = dir; - - mode_exec->init_multivar_intensities(mvdf._type, mvdo._type, config); - - ModeConfInfo & conf = mode_exec->engine.conf_info; - conf.set_field_index(field_index_f, field_index_o); - - // for multivar intensities, explicity set the level and units using stored values - // from pass1 - conf.Fcst->var_info->set_level_name(mvdf._level.c_str()); - conf.Fcst->var_info->set_units(mvdf._units.c_str()); - if (has_union_f && conf.Fcst->merge_flag == MergeType_Thresh) { - mlog << Warning << "\nModeFrontEnd::multivar_intensity_comparisons() -> " - << "Logic includes union '||' along with 'merge_flag=THRESH' " - << ". This can lead to bad results\n\n"; - } - conf.Obs->var_info->set_level_name(mvdo._level.c_str()); - conf.Obs->var_info->set_units(mvdo._units.c_str()); - if (has_union_o && conf.Obs->merge_flag == MergeType_Thresh) { - mlog << Warning << "\nModeFrontEnd::multivar_intensity_comparisons() -> " - << "Logic includes union '||' along with 'merge_flag=THRESH' " - << ". This can lead to bad results\n\n"; - } - - // - // set up data access using inputs - // - mode_exec->setup_fcst_obs_data_multivar_intensities(mvdf, mvdo); - - // - // run the mode algorithm - // - - if ( conf.quilt ) { - - do_quilt(); - - } else { - - do_straight_multivar_intensity(mvdf, mvdo, merge_f, merge_o); - - } - - // - // done - // - -#ifdef WITH_PYTHON - GP.finalize(); -#endif - return (0); -} - -/////////////////////////////////////////////////////////////////////// - -int ModeFrontEnd::run_super(const ModeConfInfo &config, - const ModeSuperObject &fsuper, - const ModeSuperObject &osuper, - GrdFileType ftype, GrdFileType otype, const Grid &grid, - bool has_union, const string &config_file, const string &dir) -{ - init(ModeExecutive::MULTIVAR_SUPER); - - mode_exec->out_dir = replace_path(default_out_dir); - mode_exec->fcst_file = "not set"; - mode_exec->obs_file = "not set"; - mode_exec->match_config_file = config_file; - mode_exec->out_dir = dir; - - mode_exec->init_multivar_intensities(ftype, otype, config); - - ModeConfInfo & conf = mode_exec->engine.conf_info; - if (has_union && (conf.Fcst->merge_flag == MergeType_Thresh || - conf.Obs->merge_flag == MergeType_Thresh)) { - mlog << Warning << "\nModeFrontEnd::run_super() -> " - << "Logic includes union '||' along with 'merge_flag=THRESH' " - << ". This can lead to bad results\n\n"; - } - - // - // set up data access using inputs - // - mode_exec->setup_fcst_obs_data_multivar_super(fsuper._simple_sd, osuper._simple_sd, grid); - - // - // run the mode algorithm - // - - if ( conf.quilt ) { - - do_quilt(); - - } else { - - do_straight_multivar_super(fsuper._merge_sd_split, osuper._merge_sd_split); - - } - - // - // done - // - -#ifdef WITH_PYTHON - GP.finalize(); -#endif - return (0); -} - -/////////////////////////////////////////////////////////////////////// - void ModeFrontEnd::do_straight() @@ -399,12 +129,9 @@ void ModeFrontEnd::do_straight() for (int index=0; indexdo_conv_thresh(index, index); - if (ptype == ModeExecutive::TRADITIONAL) { - - mode_exec->do_match_merge(); - mode_exec->process_output(); - } + mode_exec->do_conv_thresh_traditional(index, index); + mode_exec->do_match_merge_traditional(); + mode_exec->process_output_traditional(); } mode_exec->clear_internal_r_index(); @@ -420,83 +147,9 @@ void ModeFrontEnd::do_straight() /////////////////////////////////////////////////////////////////////// -void ModeFrontEnd::do_straight_multivar_intensity(const MultiVarData &mvdf, - const MultiVarData &mvdo, - const ShapeData &f_merge, - const ShapeData &o_merge) - -{ - int NCT, NCR; - - do_straight_init(NCT, NCR); - - mode_exec->clear_internal_r_index(); - - for (int index=0; indexdo_conv_thresh(index, index); - mode_exec->do_match_merge(f_merge, o_merge); - - // here replace raw data and min/max for plotting - - mode_exec->process_output(&mvdf, &mvdo); - } - - mode_exec->clear_internal_r_index(); - - // - // done - // - - return; - -} - - - -/////////////////////////////////////////////////////////////////////// - - -void ModeFrontEnd::do_straight_multivar_super(const ShapeData &f_merge, - const ShapeData &o_merge) - -{ - int NCT, NCR; - - do_straight_init(NCT, NCR); - - mode_exec->clear_internal_r_index(); - - for (int index=0; indexdo_conv_thresh(index, index); - mode_exec->do_match_merge(f_merge, o_merge); - mode_exec->process_output(); - } - - mode_exec->clear_internal_r_index(); - - // - // done - // - - return; - -} - - -/////////////////////////////////////////////////////////////////////// - - void ModeFrontEnd::do_quilt() { - if (ptype != ModeExecutive::TRADITIONAL) { - mlog << Error << "\nModeFrontend::do_quilt() -> quilting not yet implemented for multivar mode \n\n"; - exit ( 1 ); - } - - int t_index, r_index; // indices into the convolution threshold and radius arrays @@ -505,14 +158,9 @@ void ModeFrontEnd::do_quilt() for (r_index=0; r_index<(mode_exec->n_conv_radii()); ++r_index) { for (t_index=0; t_index<(mode_exec->n_conv_threshs()); ++t_index) { - - mode_exec->do_conv_thresh(r_index, t_index); - - mode_exec->do_match_merge(); - - if (ptype == ModeExecutive::TRADITIONAL) { - mode_exec->process_output(); - } + mode_exec->do_conv_thresh_traditional(r_index, t_index); + mode_exec->do_match_merge_traditional(); + mode_exec->process_output_traditional(); } } @@ -528,29 +176,13 @@ void ModeFrontEnd::do_quilt() /////////////////////////////////////////////////////////////////////// -MultiVarData *ModeFrontEnd::get_multivar_data(ModeDataType dtype) +void ModeFrontEnd::init() { - return mode_exec->get_multivar_data(dtype); -} - - -/////////////////////////////////////////////////////////////////////// - -void ModeFrontEnd::add_multivar_merge_data(MultiVarData *mvdi, ModeDataType dtype) -{ - return mode_exec->add_multivar_merge_data(mvdi, dtype); -} - -/////////////////////////////////////////////////////////////////////// - -void ModeFrontEnd::init(ModeExecutive::Processing_t p) -{ - ptype = p; - mlog << Debug(1) << "Running multivar front end for " << ModeExecutive::stype(ptype) << "\n"; + mlog << Debug(1) << "Running traditional mode front end\n"; if ( mode_exec ) { delete mode_exec; mode_exec = 0; } - mode_exec = new ModeExecutive(ptype); + mode_exec = new ModeExecutive();//ModeExecutive::TRADITIONAL); compress_level = -1; } @@ -571,20 +203,12 @@ void ModeFrontEnd::do_straight_init(int &NCT, int &NCR) const exit ( 1 ); } - - if (NCT > 1 && ptype != ModeExecutive::TRADITIONAL) { - - mlog << Error << "\nModeFrontEnd::do_straight_init() ->" - << ": multiple convolution radii and thresholds not implemented in multivar mode\n\n"; - - exit ( 1 ); - } } /////////////////////////////////////////////////////////////////////// -void ModeFrontEnd::process_command_line(const StringArray & argv, bool ismultivar) +void ModeFrontEnd::process_command_line(const StringArray & argv) { CommandLine cline; ConcatString s; @@ -597,7 +221,7 @@ void ModeFrontEnd::process_command_line(const StringArray & argv, bool ismultiva mode_exec->out_dir = replace_path(default_out_dir); // - // Check for zero arguments (note not correct for multivar mode, want to show multivar_usage + // Check for zero arguments // if(argc == 1) singlevar_usage(); @@ -609,7 +233,7 @@ void ModeFrontEnd::process_command_line(const StringArray & argv, bool ismultiva cline.set(argv); // - // Set the usage function NOTE wrong for multivar, want multivar_usage + // Set the usage function // cline.set_usage(singlevar_usage); @@ -630,103 +254,19 @@ void ModeFrontEnd::process_command_line(const StringArray & argv, bool ismultiva cline.parse(); - if (ismultivar) { - // - // Check for error. There should be 1 argument left: - // config filename - // - if(cline.n() != 1) singlevar_usage(); // wrong need multivar usage - - // - // Store the input forecast and observation file names, placeholders - // - mode_exec->fcst_file = "not set"; - mode_exec->obs_file = "not set"; - mode_exec->match_config_file = cline[0]; - - } else { - // - // Check for error. There should be three arguments left: - // forecast, observation, and config filenames - // - if(cline.n() != 3) singlevar_usage(); - - // - // Store the input forecast and observation file names - // - mode_exec->fcst_file = cline[0]; - mode_exec->obs_file = cline[1]; - mode_exec->match_config_file = cline[2]; - - } -} - - -/////////////////////////////////////////////////////////////////////// - -void ModeFrontEnd::process_command_line_for_simple_objects(const StringArray &argv, ModeDataType dtype) -{ - CommandLine cline; - ConcatString s; - const int argc = argv.n(); - - // - // Set the default output directory - // - - mode_exec->out_dir = replace_path(default_out_dir); - - // - // Check for zero arguments (note not correct for multivar mode, want to show multivar_usage - // - - if(argc == 1) singlevar_usage(); - - // - // Parse the command line into tokens // - - cline.set(argv); - - // - // Set the usage function NOTE wrong for multivar, want multivar_usage + // Check for error. There should be three arguments left: + // forecast, observation, and config filenames // - - cline.set_usage(singlevar_usage); + if(cline.n() != 3) singlevar_usage(); // - // Add the options function calls + // Store the input forecast and observation file names // + mode_exec->fcst_file = cline[0]; + mode_exec->obs_file = cline[1]; + mode_exec->match_config_file = cline[2]; - cline.add(set_config_merge_file, "-config_merge", 1); - cline.add(set_outdir, "-outdir", 1); - cline.add(set_logfile, "-log", 1); - cline.add(set_verbosity, "-v", 1); - cline.add(set_compress, "-compress", 1); - - // - // Parse the command line - // - - cline.parse(); - - // - // Check for error. There should be two arguments left: - // data and config filenames - // - if(cline.n() != 2) singlevar_usage(); - - // - // Store the file name - // - if (dtype == ModeDataType_MvMode_Fcst) { - mode_exec->fcst_file = cline[0]; - mode_exec->obs_file = "None"; - } else { - mode_exec->obs_file = cline[0]; - mode_exec->fcst_file = "None"; - } - mode_exec->match_config_file = cline[1]; } /////////////////////////////////////////////////////////////////////// diff --git a/src/tools/core/mode/mode_frontend.h b/src/tools/core/mode/mode_frontend.h index f12b40eaa0..658f90ff4c 100644 --- a/src/tools/core/mode/mode_frontend.h +++ b/src/tools/core/mode/mode_frontend.h @@ -19,10 +19,6 @@ #include #include "mode_exec.h" #include "string_array.h" -#include "multivar_data.h" -#include "mode_input_data.h" -#include "mode_data_type.h" -#include "mode_superobject.h" class ModeFrontEnd { @@ -36,75 +32,21 @@ class ModeFrontEnd { string default_out_dir; - Grid create_verification_grid(const ModeInputData &fcst, - const ModeInputData &obs, - const string &config_file, - const ModeConfInfo &config); - - - // run the multivar simple object, where there is only one input data, either forecast or obs - int create_multivar_simple_objects(const ModeConfInfo &conf, - ModeDataType dtype, const Grid &verification_grid, - const ModeInputData &input, - const string &filename, const string &config_file, - const string &outdir, - int field_index=-1, int n_files=1); - - // run the multivar simple object merge algorithm, with one input data, either forecast or obs - int create_multivar_merge_objects(const ModeConfInfo &conf, - ModeDataType dtype, const Grid &verification_grid, - const ModeInputData &input, - const string &filename, const string &config_file, - const string &outdir, - int field_index=-1, int n_files=1); // run the default single var mode interface (traditional mode) int run_traditional(const StringArray & Argv); - // run the multivar intensity algorithm, where one forecast and one obs are restricted to be within superobjects - // and the traditional mode algorithm compares them - int multivar_intensity_comparisons(const ModeConfInfo &conf, - const MultiVarData &mvdf, const MultiVarData &mvdo, - bool has_union_f, bool has_union_o, const ShapeData &merge_f, - const ShapeData &merge_o, int field_index_f, int field_index_o, - const string &fcst_filename, const string &obs_filename, - const string &config_file, const string &dir); - - - // multivar superobject interface, with no intensities - int run_super(const ModeConfInfo &conf, - const ModeSuperObject &fsuper, - const ModeSuperObject &osuper, - GrdFileType ftype, GrdFileType otype, const Grid &grid, bool has_union, - const string &config_file, const string &dir); - + void init(); // so far only implemented for traditional mode void do_quilt (); - // MODE algorithm for traditional, multivar simple, or multivar merge cases + // MODE algorithm for traditional mode void do_straight (); - // MODE algorithm when doing multivar intensities - void do_straight_multivar_intensity (const MultiVarData &mvdf, - const MultiVarData &mvdo, - const ShapeData &mergef, - const ShapeData &mergeo); - - // MODE algorithm when doing multivar super with no intensities - void do_straight_multivar_super (const ShapeData &f_merge, - const ShapeData &o_merge); - - - MultiVarData *get_multivar_data(ModeDataType dtype); - - void add_multivar_merge_data(MultiVarData *mvdi, ModeDataType dtype); - - void init(ModeExecutive::Processing_t p); void do_straight_init(int &NCT, int &NCR) const; - void process_command_line_for_simple_objects(const StringArray &, ModeDataType dtype); - void process_command_line(const StringArray &, bool is_multivar); + void process_command_line(const StringArray &); static void set_config_merge_file (const StringArray &); static void set_outdir (const StringArray &); diff --git a/src/tools/core/mode/mode_superobject.cc b/src/tools/core/mode/mode_superobject.cc index c6e24dcff3..27d1c3092f 100644 --- a/src/tools/core/mode/mode_superobject.cc +++ b/src/tools/core/mode/mode_superobject.cc @@ -1,5 +1,70 @@ using namespace std; #include "mode_superobject.h" +#include "multivar_data.h" + +//////////////////////////////////////////////////////////////////////// +static void _mask_super(const string &name, int nx, int ny, DataPlane &data) +{ + + if (nx != data.nx() || ny != data.ny()) { + mlog << Error << "\nModeSuperObject::mask_data_super() -> " << name + << " :dimensions don't match " << nx << " " << ny + << " " << data.nx() << " " << data.ny() << "\n\n"; + + exit( 1 ); + } + + int nmasked=0, nkeep=0; + + for (int x=0; x " << name + << " :dimensions don't match " << nx << " " << ny + << " " << data.nx() << " " << data.ny() << "\n\n"; + + exit( 1 ); + } + + int nmasked=0, nkeep=0; + + for (int x=0; x &mvd, BoolCalc &calc) { + _hasUnion = calc.has_union(); + // // set the BoolPlane values using the mvd content // @@ -100,3 +167,18 @@ ModeSuperObject::ModeSuperObject(bool isFcst, int n_files, bool do_clusters, delete [] simple_plane; delete [] merge_plane; } + +void ModeSuperObject::mask_data_simple(const string &name, MultiVarData &mvd) const +{ + int nx = mvd._nx; + int ny = mvd._ny; + _mask(name, nx, ny, _simple_result, mvd._simple->_sd->data); +} + + +void ModeSuperObject::mask_data_super(const string &name, const MultiVarData &mvd) +{ + int nx = mvd._nx; + int ny = mvd._ny; + _mask_super(name, nx, ny, _simple_sd.data); +} diff --git a/src/tools/core/mode/mode_superobject.h b/src/tools/core/mode/mode_superobject.h index 10151e88ed..83278b20a6 100644 --- a/src/tools/core/mode/mode_superobject.h +++ b/src/tools/core/mode/mode_superobject.h @@ -22,6 +22,8 @@ #include #include +class MultiVarData; + class ModeSuperObject { private: @@ -33,7 +35,11 @@ class ModeSuperObject { BoolCalc &calc); inline ~ModeSuperObject() {} + void mask_data_simple(const string &name, MultiVarData &mvd) const; + void mask_data_super(const string &name, const MultiVarData &mvd); + bool _isFcst; + bool _hasUnion; BoolPlane _simple_result; ShapeData _simple_sd; ShapeData _merge_sd_split; diff --git a/src/tools/core/mode/multivar_frontend.cc b/src/tools/core/mode/multivar_frontend.cc index 1512bc8a83..4d88f37df5 100644 --- a/src/tools/core/mode/multivar_frontend.cc +++ b/src/tools/core/mode/multivar_frontend.cc @@ -6,199 +6,88 @@ // ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA // *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* - -//////////////////////////////////////////////////////////////////////// - - -// for multivar mode, this is the default file -static const char mode_default_config [] = "MET_BASE/config/MODEMultivarConfig_default"; - -static const int dir_creation_mode = 0755; - //////////////////////////////////////////////////////////////////////// using namespace std; -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include "vx_util.h" -#include "file_exists.h" -#include "two_d_array.h" -#include "get_filenames.h" -#include "mode_conf_info.h" -#include "shapedata.h" -#include "interest.h" -#include "met_file.h" -#include "mode_usage.h" -#include "mode_exec.h" - -#include "combine_boolplanes.h" -#include "objects_from_netcdf.h" -#include "parse_file_list.h" -#include "mode_frontend.h" -#include "multivar_data.h" -#include "mode_input_data.h" -#include "mode_data_type.h" +#include "multivar_frontend.h" -using namespace netCDF; +#include "mode_usage.h" +#ifdef WITH_PYTHON +#include "global_python.h" +#endif //////////////////////////////////////////////////////////////////////// extern const char * const program_name; - static const char sep [] = "===================================================="; -static const char tab [] = " "; - -// this is hardwired for the multivar case, at least for now -static const bool do_clusters = false; - -static string default_out_dir = "."; - -static ModeConfInfo config; - -static string mode_path; -static string fcst_fof; -static string obs_fof; -static string config_file; -static string outdir; +static string outdir; static int compress_level = -1; -static Grid verification_grid; - -//////////////////////////////////////////////////////////////////////// - - -static void set_outdir (const StringArray &); -static void set_logfile (const StringArray &); -static void set_verbosity (const StringArray &); -static void set_compress (const StringArray &); - -static void read_input(const string &name, int index, ModeDataType type, - GrdFileType f_t, GrdFileType other_t, int shift, - vector &inputs); - -static void multivar_consistency_checks(StringArray &fcst_filenames, StringArray &obs_filenames, - BoolCalc &f_calc, BoolCalc &o_calc, int &n_fcst_files, - int &n_obs_files); - -static ConcatString set_multivar_dir(); - -static void create_verification_grid(const ModeInputData &fcst, const ModeInputData &obs); - - -static MultiVarData *create_simple_objects(ModeDataType dtype, int j, int n_files, - const string &filename, - const ConcatString &dir, - const ModeInputData &input); - -static void create_intensity_comparisons(int findex, int oindex, - const ModeSuperObject &fsuper, - const ModeSuperObject &osuper, - const ConcatString &dir, - MultiVarData &mvdf, MultiVarData &mvdo, - bool has_union_f, bool has_union_o, - const string &fcst_filename, - const string &obs_filename); - -static void process_superobjects(ModeSuperObject &fsuper, - ModeSuperObject &osuper, - int nx, int ny, const ConcatString &dir, - GrdFileType ftype, GrdFileType otype, const Grid &grid, - bool has_union); - -static void mask_data(const string &name, int nx, int ny, const BoolPlane &mask, - DataPlane &data); -static void mask_data_super(const string &name, int nx, int ny, DataPlane &data); +// for multivar mode, this is the default file +static const char mode_default_config [] = "MET_BASE/config/MODEMultivarConfig_default"; -static void read_config(const string & filename); +static const int dir_creation_mode = 0755; -static void process_command_line(const StringArray &); +static ModeExecutive *mode_exec = 0; -static int _mkdir(const char *dir); //////////////////////////////////////////////////////////////////////// - -int multivar_frontend(const StringArray & Argv) - +MultivarFrontEnd::MultivarFrontEnd() { + // this is hardwired for the multivar case, at least for now + do_clusters = false; + default_out_dir = "."; + compress_level = -1; + mode_exec = 0; +} - const int Argc = Argv.n(); - - if ( Argc < 4 ) multivar_usage(); - - int j, n_fcst_files, n_obs_files; - StringArray fcst_filenames; - StringArray obs_filenames; - BoolCalc f_calc, o_calc ; - - // set some logging related things here, used in all further processing +//////////////////////////////////////////////////////////////////////// - process_command_line(Argv); +int MultivarFrontEnd::run(const StringArray & Argv) - // read the config as fully as possible without any data reads - // (Initialize all the input fields) - - read_config(config_file); +{ - // check for length discrepencies. + // initialize - multivar_consistency_checks(fcst_filenames, obs_filenames, f_calc, o_calc, - n_fcst_files, n_obs_files); + init(Argv); mlog << Debug(2) << "\n" << sep << "\n"; - ConcatString dir = set_multivar_dir(); - // read in all the data - vector fcstInput, obsInput; - GrdFileType ft, ot; - // in the conf object, shift *can* be set independently for obs and fcst int shift = config.shift_right; for (int i=0; i mvdObs, mvdFcst; - - for (j=0; j 0) { mvdFcst[0]->checkFileTypeConsistency(*mvdi, j); } @@ -220,11 +110,14 @@ int multivar_frontend(const StringArray & Argv) mvdi->print(); } // for j - for (j=0; j 0) { mvdObs[0]->checkFileTypeConsistency(*mvdi, j); } @@ -240,11 +133,9 @@ int multivar_frontend(const StringArray & Argv) ModeSuperObject osuper(true, n_obs_files, do_clusters, mvdObs, o_calc); // - // Filter the data to within the superobjects only and do statistics by invoking mode algorithm again - // on the masked data pairs + // Filter the data to within the superobjects only and do statistics by invoking mode + // algorithm again on the masked data pairs // - bool f_has_union = f_calc.has_union(); - bool o_has_union = o_calc.has_union(); for (int k=0; k_nx; - int ny = mvdFcst[0]->_ny; - GrdFileType ftype = mvdFcst[0]->_type; - GrdFileType otype = mvdObs[0]->_type; - Grid grid = *(mvdFcst[0]->_grid); + if (config.fcst_multivar_compare_index.n() <= 0) { - // here run one more time using superobjects as input + process_superobjects(fsuper, osuper, *mvdFcst[0], *mvdObs[0]); + } + + // + // done + // + return (0); +} - bool has_union = f_calc.has_union() || o_calc.has_union(); +//////////////////////////////////////////////////////////////////////// - process_superobjects(fsuper, osuper, nx, ny, dir, ftype, otype, grid, has_union); +MultivarFrontEnd::~MultivarFrontEnd() +{ + if ( mode_exec ) { + delete mode_exec; mode_exec = 0; } - - // free up memory - for (j=0; jgrid(); + GrdFileType ft = f->file_type(); + + //? + f->set_shift_right(shift); + + // update config now that we know file type (this sets Fcst to index i) + DataPlane dp; + + if (type == ModeDataType_MvMode_Fcst) { + config.process_config_field(ft, other_t, type, index); + f->data_plane(*(config.Fcst->var_info), dp); + fcstInput.push_back(ModeInputData(name, dp, g, ft)); + } else { + config.process_config_field(other_t, ft, type, index); + f->data_plane(*(config.Obs->var_info), dp); + obsInput.push_back(ModeInputData(name, dp, g, ft)); + } + + delete f; +} + //////////////////////////////////////////////////////////////////////// +void MultivarFrontEnd::create_verif_grid() +{ + mlog << Debug(2) << "\n creating the verification grid \n" << sep << "\n"; + + _init_exec(ModeExecutive::TRADITIONAL, "None", "None"); + // mode_exec->init_multivar_verif_grid(fcstInput[0]._dataPlane, + // obsInput[0]._dataPlane, config); + // ModeConfInfo & conf = mode_exec->engine.conf_info; + // conf.set_field_index(0); + mode_exec->setup_verification_grid(fcstInput[0], obsInput[0], config); + verification_grid = mode_exec->grid; + delete mode_exec; mode_exec = 0; +} -void read_config(const string & filename) +//////////////////////////////////////////////////////////////////////// +MultiVarData *MultivarFrontEnd::create_simple_objects(ModeDataType dtype, int j, + int n_files, + const string &filename, + const ModeInputData &input) { + // + // create simple non merged objects + // + _simple_objects(ModeExecutive::MULTIVAR_SIMPLE, dtype, j, n_files, + filename, input); + MultiVarData *mvdi = mode_exec->get_multivar_data(dtype); + delete mode_exec; mode_exec = 0; - ConcatString path; + // + // create simple merged objects + // + _simple_objects(ModeExecutive::MULTIVAR_SIMPLE_MERGE, dtype, j, n_files, + filename, input); + mode_exec->add_multivar_merge_data(mvdi, dtype); + delete mode_exec; mode_exec = 0; + return mvdi; +} - path = replace_path(mode_default_config); +//////////////////////////////////////////////////////////////////////// - config.read_config(path.c_str(), filename.c_str()); +void +MultivarFrontEnd::create_intensity_comparisons(int findex, int oindex, + const ModeSuperObject &fsuper, + const ModeSuperObject &osuper, + MultiVarData &mvdf, MultiVarData &mvdo, + const string &fcst_filename, + const string &obs_filename) +{ - // process the config except for the fields - config.process_config_except_fields(); + // mask the input data to be valid only inside the simple super objects + fsuper.mask_data_simple("Fcst", mvdf); + osuper.mask_data_simple("Obs", mvdo); - // done once here, used for all data - // what is this, command line overrides config? look deeper.. remove from exec - // except traditional mode - if (compress_level >= 0) config.nc_info.set_compress_level(compress_level); - // from within mode_exec: - // engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); + mlog << Debug(1) << "Running mvmode intensity comparisions \n\n"; + _init_exec(ModeExecutive::MULTIVAR_INTENSITY, fcst_filename, obs_filename); + mode_exec->init_multivar_intensities(mvdf._type, mvdo._type, config); - return; + ModeConfInfo & conf = mode_exec->engine.conf_info; + conf.set_field_index(findex, oindex); -} + // for multivar intensities, explicity set the level and units using stored values + // from pass1 + conf.Fcst->var_info->set_level_name(mvdf._level.c_str()); + conf.Fcst->var_info->set_units(mvdf._units.c_str()); + if (fsuper._hasUnion && conf.Fcst->merge_flag == MergeType_Thresh) { + mlog << Warning << "\nModeFrontEnd::multivar_intensity_comparisons() -> " + << "Logic includes union '||' along with 'merge_flag=THRESH' " + << ". This can lead to bad results\n\n"; + } + conf.Obs->var_info->set_level_name(mvdo._level.c_str()); + conf.Obs->var_info->set_units(mvdo._units.c_str()); + if (osuper._hasUnion && conf.Obs->merge_flag == MergeType_Thresh) { + mlog << Warning << "\nModeFrontEnd::multivar_intensity_comparisons() -> " + << "Logic includes union '||' along with 'merge_flag=THRESH' " + << ". This can lead to bad results\n\n"; + } + + // + // set up data access using inputs + // + mode_exec->setup_multivar_fcst_obs_data_intensities(mvdf, mvdo); + + // + // run the mode algorithm for multivar intensities + // + _intensity_compare_mode_algorithm(mvdf, mvdo, fsuper, osuper); + delete mode_exec; mode_exec = 0; +} //////////////////////////////////////////////////////////////////////// +void MultivarFrontEnd::process_superobjects(ModeSuperObject &fsuper, + ModeSuperObject &osuper, + const MultiVarData &mvdf, + const MultiVarData &mvdo) +{ + mlog << Debug(1) << "Running superobject mode \n\n"; + + // set the data to 0 inside superobjects and missing everywhere else + + fsuper.mask_data_super("FcstSimple", mvdf); + osuper.mask_data_super("ObsSimple", mvdo); -void process_command_line(const StringArray & argv) + _init_exec(ModeExecutive::MULTIVAR_SUPER, "None", "None"); + mode_exec->init_multivar_intensities(mvdf._type, mvdo._type, config); + + ModeConfInfo & conf = mode_exec->engine.conf_info; + if ((fsuper._hasUnion || osuper._hasUnion) && + (conf.Fcst->merge_flag == MergeType_Thresh || + conf.Obs->merge_flag == MergeType_Thresh)) { + mlog << Warning << "\nModeFrontEnd::run_super() -> " + << "Logic includes union '||' along with 'merge_flag=THRESH' " + << ". This can lead to bad results\n\n"; + } + + // + // set up data access using inputs + // + mode_exec->setup_multivar_fcst_obs_data_super(fsuper._simple_sd, osuper._simple_sd, + *mvdf._grid); + + // run the mode algorithm + _superobject_mode_algorithm(fsuper, osuper); + + delete mode_exec; mode_exec = 0; +} + +//////////////////////////////////////////////////////////////////////// + +void MultivarFrontEnd::_process_command_line(const StringArray & argv) { @@ -411,57 +454,45 @@ void process_command_line(const StringArray & argv) // should be 3 arguments left // - fcst_fof = cline[0]; obs_fof = cline[1]; config_file = cline[2]; - return; } //////////////////////////////////////////////////////////////////////// -void read_input(const string &name, int index, ModeDataType type, - GrdFileType f_t, GrdFileType other_t, int shift, - vector &inputs) - { - Met2dDataFileFactory mtddf_factory; - Met2dDataFile *f = mtddf_factory.new_met_2d_data_file(name.c_str(), f_t); - if (!f) { - mlog << Error << "\nTrouble reading fcst file \"" - << name << "\"\n\n"; - exit(1); - } - Grid g = f->grid(); - GrdFileType ft = f->file_type(); - - //? - f->set_shift_right(shift); - - // update config now that we know file type (this sets Fcst to index i) - DataPlane dp; - - if (type == ModeDataType_MvMode_Fcst) { - config.process_config_field(ft, other_t, type, index); - // need to have set the var_info! - f->data_plane(*(config.Fcst->var_info), dp); - } else { - config.process_config_field(other_t, ft, type, index); - // need to have set the var_info! - f->data_plane(*(config.Obs->var_info), dp); - } - - inputs.push_back(ModeInputData(name, dp, g, ft)); - delete f; - } - + +void MultivarFrontEnd::_read_config(const string & filename) + +{ + + ConcatString path; + + path = replace_path(mode_default_config); + + config.read_config(path.c_str(), filename.c_str()); + + // process the config except for the fields + config.process_config_except_fields(); + + // done once here, used for all data + // what is this, command line overrides config? look deeper.. remove from exec + // except traditional mode + if (compress_level >= 0) config.nc_info.set_compress_level(compress_level); + // from within mode_exec: + // engine.conf_info.nc_info.compress_level = engine.conf_info.get_compression_level(); + + + return; + +} + //////////////////////////////////////////////////////////////////////// -void multivar_consistency_checks(StringArray &fcst_filenames, StringArray &obs_filenames, - BoolCalc &f_calc, BoolCalc &o_calc, int &n_fcst_files, - int &n_obs_files) +void MultivarFrontEnd::_setup_inputs() { // // make sure the multivar logic programs are in the config file @@ -547,234 +578,190 @@ void multivar_consistency_checks(StringArray &fcst_filenames, StringArray &obs_f //////////////////////////////////////////////////////////////////////// -ConcatString set_multivar_dir() +void MultivarFrontEnd::_set_output_path() { - ConcatString dir; int status; - dir.clear(); + output_path.clear(); // no longer want numbered subdirectories - if ( outdir.length() > 0 ) dir << outdir; + if ( outdir.length() > 0 ) output_path << outdir; // // test to see of the output directory for this // mode runs exists, and if not, create it // - if ( ! directory_exists(dir.c_str()) ) { + if ( ! directory_exists(output_path.c_str()) ) { mlog << Debug(2) << program_name << ": creating output directory \"" - << dir << "\"\n\n"; + << output_path << "\"\n\n"; - status = _mkdir(dir.c_str()); + status = _mkdir(output_path.c_str()); if ( status < 0 ) { mlog << Error << "\nset_multivar_dir() ->" << " unable to create output directory \"" - << dir << "\"\n\n"; + << output_path << "\"\n\n"; exit ( 1 ); } } - return dir; } //////////////////////////////////////////////////////////////////////// -void create_verification_grid(const ModeInputData &fcst, - const ModeInputData &obs) +int MultivarFrontEnd::_mkdir(const char *dir) { - ModeFrontEnd *frontend = new ModeFrontEnd; - verification_grid = frontend->create_verification_grid(fcst, obs, - config_file, - config); - delete frontend; + char tmp[256]; + char *p = NULL; + size_t len; + + snprintf(tmp, sizeof(tmp),"%s",dir); + len = strlen(tmp); + if (tmp[len - 1] == '/') + tmp[len - 1] = 0; + for (p = tmp + 1; *p; p++) + if (*p == '/') { + *p = 0; + string s = tmp; + if (s != ".") { + if (mkdir(tmp, dir_creation_mode) < 0) { + mlog << Error << "\n_mkdir() -> Error making " << tmp << "\n"; + return -1; + } + } + *p = '/'; + } + + return (mkdir(tmp, dir_creation_mode)); } //////////////////////////////////////////////////////////////////////// -MultiVarData *create_simple_objects(ModeDataType dtype, int j, int n_files, - const string &filename, - const ConcatString &dir, - const ModeInputData &input) +void MultivarFrontEnd::_simple_objects(ModeExecutive::Processing_t p, + ModeDataType dtype, + int j, int n_files, const string &filename, + const ModeInputData &input) { - ModeFrontEnd *frontend = new ModeFrontEnd; - int status = - frontend->create_multivar_simple_objects(config, dtype, verification_grid, input, - filename, config_file, dir, - j, n_files); - MultiVarData *mvdi = frontend->get_multivar_data(dtype); - delete frontend; + if (dtype == ModeDataType_MvMode_Fcst) { + _init_exec(p, filename, "None"); + mode_exec->init_multivar_simple(j, n_files, dtype, config); + mode_exec->setup_multivar_fcst_data(verification_grid, input); + } else { + _init_exec(p, "None", filename); + mode_exec->init_multivar_simple(j, n_files, dtype, config); + mode_exec->setup_multivar_obs_data(verification_grid, input); + } + + _simple_mode_algorithm(p); +} - // - // create simple merge objects - // +//////////////////////////////////////////////////////////////////////// - frontend = new ModeFrontEnd; - status = frontend->create_multivar_merge_objects(config, dtype, verification_grid, input, - filename, config_file, dir, - j, n_files); +void MultivarFrontEnd::_init_exec(ModeExecutive::Processing_t p, + const string &ffile, + const string &ofile) +{ + mlog << Debug(1) << "Running multivar front end for " << ModeExecutive::stype(p) << "\n"; - // add the merge results to the mvdi object - frontend->add_multivar_merge_data(mvdi, dtype); - delete frontend; + if ( mode_exec ) { delete mode_exec; mode_exec = 0; } - return mvdi; + mode_exec = new ModeExecutive(); + // compress_level = -1; + mode_exec->fcst_file = ffile; + mode_exec->obs_file = ofile; + mode_exec->match_config_file = config_file; + mode_exec->out_dir = output_path; } //////////////////////////////////////////////////////////////////////// void -create_intensity_comparisons(int findex, int oindex, - const ModeSuperObject &fsuper, - const ModeSuperObject &osuper, - const ConcatString &dir, - MultiVarData &mvdf, MultiVarData &mvdo, - bool has_union_f, bool has_union_o, - const string &fcst_filename, - const string &obs_filename) +MultivarFrontEnd::_superobject_mode_algorithm(const ModeSuperObject &fsuper, + const ModeSuperObject &osuper) { - - // mask the input data to be valid only inside the simple super objects - int nx = mvdf._nx; - int ny = mvdf._ny; - - mask_data("Fcst", nx, ny, fsuper._simple_result, mvdf._simple->_sd->data); - mask_data("Obs", nx, ny, osuper._simple_result, mvdo._simple->_sd->data); - - mlog << Debug(1) << "Running mvmode intensity comparisions \n\n"; - - ModeFrontEnd *frontend = new ModeFrontEnd; - int status = frontend->multivar_intensity_comparisons(config, mvdf, mvdo, has_union_f, - has_union_o, - fsuper._merge_sd_split, - osuper._merge_sd_split, - findex, - oindex, - fcst_filename, - obs_filename, - config_file, dir); - delete frontend; + _mode_algorithm_init(); + mode_exec->clear_internal_r_index(); + mode_exec->do_conv_thresh_multivar_super(); + mode_exec->do_match_merge_multivar(fsuper._merge_sd_split, osuper._merge_sd_split, + ModeExecutive::MULTIVAR_SUPER); + mode_exec->process_output_multivar_super(); + mode_exec->clear_internal_r_index(); +#ifdef WITH_PYTHON + GP.finalize(); + #endif } //////////////////////////////////////////////////////////////////////// -void process_superobjects(ModeSuperObject &fsuper, - ModeSuperObject &osuper, - int nx, int ny, const ConcatString &dir, - GrdFileType ftype, GrdFileType otype, - const Grid &grid, bool has_union) +void +MultivarFrontEnd::_intensity_compare_mode_algorithm(const MultiVarData &mvdf, + const MultiVarData &mvdo, + const ModeSuperObject &fsuper, + const ModeSuperObject &osuper) { - mlog << Debug(1) << "Running superobject mode \n\n"; + _mode_algorithm_init(); + mode_exec->do_conv_thresh_multivar_intensity_compare(); + mode_exec->do_match_merge_multivar(fsuper._merge_sd_split, osuper._merge_sd_split, + ModeExecutive::MULTIVAR_INTENSITY); + // here replace raw data and min/max for plotting + mode_exec->process_output_multivar_intensity_compare(&mvdf, &mvdo); + mode_exec->clear_internal_r_index(); +#ifdef WITH_PYTHON + GP.finalize(); + #endif +} - // set the data to 0 inside superobjects and missing everywhere else - mask_data_super("FcstSimple", nx, ny, fsuper._simple_sd.data); - mask_data_super("ObsSimple", nx, ny, osuper._simple_sd.data); - - - ModeFrontEnd *frontend = new ModeFrontEnd; - - int status = frontend->run_super(config, fsuper, osuper, - ftype, otype, grid, has_union, - config_file, dir); - delete frontend; -} - //////////////////////////////////////////////////////////////////////// -void mask_data(const string &name, int nx, int ny, const BoolPlane &bp, DataPlane &data) +void MultivarFrontEnd::_simple_mode_algorithm(ModeExecutive::Processing_t p) { - - if (nx != data.nx() || ny != data.ny()) { - mlog << Error << "\nmask_data() -> " << name - << " :dimensions don't match " << nx << " " << ny - << " " << data.nx() << " " << data.ny() << "\n\n"; - - exit( 1 ); - } - - int nmasked=0, nkeep=0; + _mode_algorithm_init(); + mode_exec->clear_internal_r_index(); + mode_exec->do_conv_thresh_multivar_simple(p); + mode_exec->clear_internal_r_index(); - for (int x=0; xengine.conf_info; + if ( conf.quilt ) { + mlog << Error << "\nMultiVarFontend::mode_algorithm() -> " + << "quilting not yet implemented for multivar mode \n\n"; + exit ( 1 ); } - - mlog << Debug(1) << name << " superobject masking.." - << nkeep << " points of " - << nmasked + nkeep << " in superobjects\n"; -} + int NCT = conf.n_conv_threshs(); + int NCR = conf.n_conv_radii(); + if ( NCT != NCR ) { -//////////////////////////////////////////////////////////////////////// -void mask_data_super(const string &name, int nx, int ny, DataPlane &data) -{ + mlog << Error << "\nMultivarFrontEnd::_mode_algorithm_init() ->" + << "all convolution radius and threshold arrays must have the same number of elements\n\n"; - if (nx != data.nx() || ny != data.ny()) { - mlog << Error << "\nmask_data_super() -> " << name - << " :dimensions don't match " << nx << " " << ny - << " " << data.nx() << " " << data.ny() << "\n\n"; + exit ( 1 ); - exit( 1 ); } - int nmasked=0, nkeep=0; - - for (int x=0; x 1) { - for (int y=0; y" + << ": multiple convolution radii and thresholds not implemented in multivar mode\n\n"; - if(is_bad_data(data.get(x,y))) { - nmasked ++; - } else { - data.set(0.0, x, y); - nkeep ++; - } - } + exit ( 1 ); } - - mlog << Debug(1) << name << " superobject masking.." - << nkeep << " points of " - << nmasked + nkeep << " in superobjects\n"; } -//////////////////////////////////////////////////////////////////////// - -int _mkdir(const char *dir) -{ - char tmp[256]; - char *p = NULL; - size_t len; - snprintf(tmp, sizeof(tmp),"%s",dir); - len = strlen(tmp); - if (tmp[len - 1] == '/') - tmp[len - 1] = 0; - for (p = tmp + 1; *p; p++) - if (*p == '/') { - *p = 0; - string s = tmp; - if (s != ".") { - if (mkdir(tmp, dir_creation_mode) < 0) { - mlog << Error << "\n_mkdir() -> Error making " << tmp << "\n"; - return -1; - } - } - *p = '/'; - } - return (mkdir(tmp, dir_creation_mode)); -} + diff --git a/src/tools/core/mode/multivar_frontend.h b/src/tools/core/mode/multivar_frontend.h new file mode 100644 index 0000000000..a518bc5d31 --- /dev/null +++ b/src/tools/core/mode/multivar_frontend.h @@ -0,0 +1,106 @@ +// ** Copyright UCAR (c) 1992 - 2023 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + + +//////////////////////////////////////////////////////////////////////// + + +#ifndef __MULTIVAR_FRONTEND_H__ +#define __MULTIVAR_FRONTEND_H__ + + +//////////////////////////////////////////////////////////////////////// + +#include +#include "mode_conf_info.h" +#include "two_d_array.h" +#include "bool_calc.h" +#include "multivar_data.h" +#include "mode_superobject.h" +#include "mode_input_data.h" +#include "mode_exec.h" + +class MultivarFrontEnd { + +private: + + int n_fcst_files, n_obs_files; + StringArray fcst_filenames; + StringArray obs_filenames; + BoolCalc f_calc, o_calc ; + vector fcstInput, obsInput; + vector mvdFcst, mvdObs; + string fcst_fof; + string obs_fof; + + void _process_command_line(const StringArray &); + void _read_config(const string & filename); + void _setup_inputs(); + void _set_output_path(); + int _mkdir(const char *dir); + void _simple_objects(ModeExecutive::Processing_t p, ModeDataType dtype, + int j, int n_files, const string &filename, + const ModeInputData &input); + void _init_exec(ModeExecutive::Processing_t p, const string &ffile, const string &ofile); + void _superobject_mode_algorithm(const ModeSuperObject &fsuper, const ModeSuperObject &osuper); + void _intensity_compare_mode_algorithm(const MultiVarData &mvdf, const MultiVarData &mvdo, + const ModeSuperObject &fsuper, const ModeSuperObject &osuper); + void _simple_mode_algorithm(ModeExecutive::Processing_t p); + void _mode_algorithm_init() const; + +public: + + bool do_clusters; + string default_out_dir; + ModeConfInfo config; + ConcatString output_path; + string mode_path; + string config_file; + Grid verification_grid; + + MultivarFrontEnd(); + + ~MultivarFrontEnd(); + + + int run(const StringArray & Argv); + void init(const StringArray & Argv); + + static void set_outdir (const StringArray &); + static void set_logfile (const StringArray &); + static void set_verbosity (const StringArray &); + static void set_compress (const StringArray &); + + void read_input(const string &name, int index, ModeDataType type, + GrdFileType f_t, GrdFileType other_t, int shift); + + + void create_verif_grid(void); + + MultiVarData *create_simple_objects(ModeDataType dtype, int j, int n_files, + const string &filename, + const ModeInputData &input); + + void create_intensity_comparisons(int findex, int oindex, + const ModeSuperObject &fsuper, + const ModeSuperObject &osuper, + MultiVarData &mvdf, MultiVarData &mvdo, + const string &fcst_filename, + const string &obs_filename); + + void process_superobjects(ModeSuperObject &fsuper, + ModeSuperObject &osuper, + const MultiVarData &mvdf, + const MultiVarData &mvdo); + +}; + + +#endif /* __MULTIVAR_FRONT_END_H__ */ + + +///////////////////////////////////////////////////////////////////////// From 992f8cd37d17c6d74f08d7c25ebb07529fc90a15 Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Tue, 2 Jan 2024 22:36:13 +0000 Subject: [PATCH 03/14] documentation changes to go with changes to percentiles and data_type --- docs/Users_Guide/mode.rst | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/Users_Guide/mode.rst b/docs/Users_Guide/mode.rst index 5a6503f0f5..becf20ed98 100644 --- a/docs/Users_Guide/mode.rst +++ b/docs/Users_Guide/mode.rst @@ -118,6 +118,28 @@ The **multivar_intensity_compare_fcst** and **multivar_intensity_compare_obs** c When regridding to the FCST or OBS field (e.g. to_grid = FCST), the first field of the field array is used from the forecast and observation field dictionaries, respectively. All regridding is then done to that grid. Other regrid options described in :ref:`regrid` can also be used as normal. +"file_type" can be set independently for each input in multivariate mode. If not set for an input, MET uses file names and file content to determine the type. + +When setting a threshold to a percentile, some choices require both an observation input and a forecast input. When this is the case, it's assumed the indices match, so for example if forecast input 1 has such a percentile setting, then observation input 1 will be used to compute the percentile. Percentiles in which this will happen are: + +* SFP in an observation input + * The matching forecast input will be used to determine the threshold. + e.g. ">SFP33.3" in the 2nd observation input means greater than 33.3-rd percentile of the 2nd forecast input will be used as the threshold for that observation input. +* SOP in a forecast input + * The matching observation input will be used to determine the threshold. + e.g. ">SOP33.3" in the 2nd forecast input means greater than 33.3-rd percentile of the 2nd observation input will be used as the threshold for that forecast input. + +* "==FBIAS" in an observation input. + * e.g. "==FBIAS1" in an observation input to automatically de-bias the data, using a simple threshold in the matching forecast input. For example, + when observation input 3 has "==FBIAS1", and forecast input 3 has ">5.0", MET applies the >5.0 threshold to the forecast and then chooses an + observation threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0. + +* "==FBIAS" in a forecast input. + * e.g. "==FBIAS1" in a forecast input to automatically de-bias the data, using a simple threshold in the matching observation input. For example, + when forecast input 2 has "==FBIAS1", and observation input 2 has ">5.0", MET applies the >5.0 threshold to the observation and then chooses a + forecast threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0. + + Practical Information ===================== From 4e3b53646d97f8f8de031cdca1e42b643809c91e Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Tue, 2 Jan 2024 23:14:05 +0000 Subject: [PATCH 04/14] Cleanup and bugfix to doc --- src/libcode/vx_shapedata/engine.cc | 16 ++- src/libcode/vx_shapedata/engine.h | 14 +-- src/libcode/vx_shapedata/mode_conf_info.cc | 119 ++++++++++++--------- src/libcode/vx_shapedata/mode_input_data.h | 7 +- src/tools/core/mode/mode_exec.cc | 21 ++-- src/tools/core/mode/mode_exec.h | 43 ++++++-- src/tools/core/mode/multivar_data.cc | 22 +--- src/tools/core/mode/multivar_data.h | 7 +- src/tools/core/mode/multivar_frontend.cc | 21 ++-- 9 files changed, 134 insertions(+), 136 deletions(-) diff --git a/src/libcode/vx_shapedata/engine.cc b/src/libcode/vx_shapedata/engine.cc index dd041dc9c1..c5bcdf9b46 100644 --- a/src/libcode/vx_shapedata/engine.cc +++ b/src/libcode/vx_shapedata/engine.cc @@ -965,9 +965,7 @@ void ModeFuzzyEngine::do_obs_merging(const char *default_config, /////////////////////////////////////////////////////////////////////// -void ModeFuzzyEngine::do_fcst_merging(const char *default_config, - const char *merge_config, - const ShapeData &merge_data) +void ModeFuzzyEngine::do_fcst_merging(const ShapeData &merge_data) { if(need_fcst_thresh) do_fcst_thresholding(); @@ -986,7 +984,7 @@ void ModeFuzzyEngine::do_fcst_merging(const char *default_config, if(conf_info.Fcst->merge_flag == MergeType_Both || conf_info.Fcst->merge_flag == MergeType_Engine) - do_fcst_merge_engine(default_config, merge_config); + do_fcst_merge_engine("", ""); // // Done @@ -1001,14 +999,12 @@ void ModeFuzzyEngine::do_fcst_merging(const char *default_config, /////////////////////////////////////////////////////////////////////// -void ModeFuzzyEngine::do_obs_merging(const char *default_config, - const char *merge_config, - const ShapeData &merge_data) +void ModeFuzzyEngine::do_obs_merging(const ShapeData &merge_data) { if(need_obs_thresh) do_obs_thresholding(); if(!need_obs_merge) return; - + if (obs_thresh->data.nx() != merge_data.data.nx() || obs_thresh->data.ny() != merge_data.data.ny()) { mlog << Error << "\nModeFuzzyEngine::do_obs_merging() -> " @@ -1023,8 +1019,8 @@ void ModeFuzzyEngine::do_obs_merging(const char *default_config, if(conf_info.Obs->merge_flag == MergeType_Both || conf_info.Obs->merge_flag == MergeType_Engine) - do_obs_merge_engine(default_config, merge_config); - + do_obs_merge_engine("", ""); + // // Done // diff --git a/src/libcode/vx_shapedata/engine.h b/src/libcode/vx_shapedata/engine.h index 8bc5995dfb..0b8241110a 100644 --- a/src/libcode/vx_shapedata/engine.h +++ b/src/libcode/vx_shapedata/engine.h @@ -215,16 +215,18 @@ class ModeFuzzyEngine { void do_fcst_merging(); void do_obs_merging(); + // traditional version void do_fcst_merging(const char *default_config, const char *merge_config); - void do_fcst_merging(const char *default_config, - const char *merge_config, - const ShapeData &merge_data); + + // multivar version + void do_fcst_merging(const ShapeData &merge_data); + + // traditional version void do_obs_merging(const char *default_config, const char *merge_config); - void do_obs_merging(const char *default_config, - const char *merge_config, - const ShapeData &merge_data); + // multivar version + void do_obs_merging(const ShapeData &merge_data); void do_matching(); diff --git a/src/libcode/vx_shapedata/mode_conf_info.cc b/src/libcode/vx_shapedata/mode_conf_info.cc index 615e3f90eb..9008152e6f 100644 --- a/src/libcode/vx_shapedata/mode_conf_info.cc +++ b/src/libcode/vx_shapedata/mode_conf_info.cc @@ -680,105 +680,112 @@ void ModeConfInfo::config_set_all_percentile_thresholds(const std::vector &odata) { - // for each forecast input, it's either not a percentile threshold, is a simple (single input) percentile, - // a frequency bias percentile threshold, or a climatology percentile (we don't have climatology). - // as a complication we have both conv_thresh and merge_thresh, so what if they are different? - // right now I simply go with freaquency bias as highest priority + // for each forecast input, it's either not a percentile threshold, + // is a simple (single input) percentile, is a frequency bias percentile threshold, or is a + // climatology percentile (we don't have climatology). + // + // As a complication we have both conv_thresh and merge_thresh, so what if they are different? + // right now I simply go with frequency bias as highest priority + + // indices of forecast and obs inputs that have frequency bias percentile thresholding vector fcst_freq, obs_freq; + + // indices (common to forecast and obs) that require both inputs (fcst and obs) + // which is either frequency bias, or sample obs with a forecast input, or sample fcst + // with an obs input + vector indices_with_both; + for (int j=0; j= N_fields_o) { + mlog << Error << "\nModeConfInfo::config_set_all_percentile_thresholds\n" + << " SOP Thresholding on fcst index " << j+1 + << " out of range of obs " << N_fields_o + 1 << "\n\n"; + exit ( 1 ); + } + // deal with this later + indices_with_both.push_back(j); break; case perc_thresh_freq_bias: // currently expect matching index in obs and fcst, so check to make sure not both if (j >= N_fields_o) { mlog << Error << "\nModeConfInfo::config_set_all_percentile_thresholds\n" - << " Frequency bias Thresholding on fcst index " << j+1 + << " FBIAS Thresholding on fcst index " << j+1 << " out of range of obs " << N_fields_o + 1 << "\n\n"; exit ( 1 ); } - // save for later + fcst_freq.push_back(j); + indices_with_both.push_back(j); break; default: break; } - - if (need_perc) { - data_type = ModeDataType_MvMode_Fcst; - set_field_index(j); - set_perc_thresh(fdata[j]._dataPlane); - } - if (need_freq) { - fcst_freq.push_back(j); - } } - for (int j=0; j= N_fields_f) { + mlog << Error << "\nModeConfInfo::config_set_all_percentile_thresholds\n" + << " SFP Thresholding on obs index " << j+1 + << " out of range of fcst " << N_fields_f + 1 << "\n\n"; + exit ( 1 ); + } + // deal with this later + if (find(indices_with_both.begin(), indices_with_both.end(), j) == indices_with_both.end()) { + indices_with_both.push_back(j); + } break; case perc_thresh_freq_bias: // currently expect matching index in obs and fcst, so check to make sure not both if (j >= N_fields_f) { mlog << Error << "\nModeConfInfo::config_set_all_percentile_thresholds\n" - << " Frequency bias Thresholding on obs index " << j+1 + << " FBIAS Thresholding on obs index " << j+1 << " out of range of fcst " << N_fields_f + 1 << "\n\n"; exit ( 1 ); } - // save for later + // deal with this later + obs_freq.push_back(j); + if (find(indices_with_both.begin(), indices_with_both.end(), j) == indices_with_both.end()) { + indices_with_both.push_back(j); + } break; default: break; } - - if (need_perc) { - data_type = ModeDataType_MvMode_Obs; - set_field_index(j); - set_perc_thresh(odata[j]._dataPlane); - } - if (need_freq) { - obs_freq.push_back(j); - } } - // make sure no overlap in fcst/obs frequencies + // make sure no overlap in fcst/obs frequencies, as that's a no no for (size_t i=0; i #include "data_plane.h" #include "grid_base.h" -#include "data_file_type.h" //////////////////////////////////////////////////////////////////////// @@ -26,16 +25,14 @@ class ModeInputData { public: - ModeInputData(const std::string &name, const DataPlane &dp, - const Grid &g, const GrdFileType t) : - _name(name), _dataPlane(dp), _grid(g), _fileType(t) { } + ModeInputData(const std::string &name, const DataPlane &dp, const Grid &g) : + _name(name), _dataPlane(dp), _grid(g) {} ~ModeInputData() {} std::string _name; DataPlane _dataPlane; Grid _grid; - GrdFileType _fileType; }; diff --git a/src/tools/core/mode/mode_exec.cc b/src/tools/core/mode/mode_exec.cc index 2f44e944c0..c3dc975fda 100644 --- a/src/tools/core/mode/mode_exec.cc +++ b/src/tools/core/mode/mode_exec.cc @@ -35,7 +35,6 @@ static const char * cts_str[n_cts] = {"RAW", "OBJECT"}; static const char program_name [] = "mode"; static const char * default_config_filename = "MET_BASE/config/MODEConfig_default"; -static const char * default_multivar_config_filename = "MET_BASE/config/MODEMultivarConfig_default"; // took this out of the do_conv_thresh() method static int local_r_index = -1; @@ -151,7 +150,7 @@ void ModeExecutive::init_traditional(int n_files) R_index = T_index = 0; - conf_read(default_config_filename); + conf_read(); // Get the forecast and observation file types from config, if present ftype = parse_conf_file_type(engine.conf_info.conf.lookup_dictionary(conf_key_fcst)); @@ -223,7 +222,7 @@ void ModeExecutive::init_multivar_simple(int j, int n_files, ModeDataType dtype, /////////////////////////////////////////////////////////////////////// -void ModeExecutive::init_multivar_intensities(GrdFileType ftype, GrdFileType otype, const ModeConfInfo &conf) +void ModeExecutive::init_multivar_intensities(const ModeConfInfo &conf) { @@ -430,7 +429,6 @@ void ModeExecutive::setup_multivar_fcst_data(const Grid &verification_grid, double fmin, fmax; Fcst_sd.clear(); - ftype = input._fileType; Fcst_sd.data = input._dataPlane; grid = verification_grid; @@ -494,7 +492,6 @@ void ModeExecutive::setup_multivar_obs_data(const Grid &verification_grid, double omin, omax; Obs_sd.clear(); - otype = input._fileType; // Read the gridded data from the input observation file Obs_sd.data = input._dataPlane; @@ -973,8 +970,7 @@ void ModeExecutive::do_merging_multivar(const ShapeData &f_merge, // Do the forecast merging - engine.do_fcst_merging(default_config_file.c_str(), merge_config_file.c_str(), - f_merge); + engine.do_fcst_merging(f_merge); mlog << Debug(2) << "Performing merging (" @@ -983,8 +979,7 @@ void ModeExecutive::do_merging_multivar(const ShapeData &f_merge, // Do the observation merging - engine.do_obs_merging(default_config_file.c_str(), merge_config_file.c_str(), - o_merge); + engine.do_obs_merging(o_merge); mlog << Debug(2) << "Remaining: " << engine.n_fcst << " forecast objects " @@ -1598,7 +1593,8 @@ MultiVarData *ModeExecutive::get_multivar_data(ModeDataType dtype) obs_magic_string = engine.conf_info.Obs->var_info->magic_str().c_str(); // replace forward slashes with underscores to prevent new directories replace(obs_magic_string.begin(), obs_magic_string.end(), '/', '_'); - mvd->init(dtype, obs_magic_string, grid, otype, ounits, olevel, data_min, data_max); + mvd->init(dtype, obs_magic_string, grid, + ounits, olevel, data_min, data_max); mvd->set_obj(engine.obs_split, simple); mvd->set_raw(engine.obs_raw, simple); mvd->set_shapedata(Obs_sd, simple); @@ -1609,7 +1605,8 @@ MultiVarData *ModeExecutive::get_multivar_data(ModeDataType dtype) fcst_magic_string = engine.conf_info.Fcst->var_info->magic_str().c_str(); // replace forward slashes with underscores to prevent new directories replace(fcst_magic_string.begin(), fcst_magic_string.end(), '/', '_'); - mvd->init(dtype, fcst_magic_string, grid, ftype, funits, flevel, data_min, data_max); + mvd->init(dtype, fcst_magic_string, grid, + funits, flevel, data_min, data_max); mvd->set_obj(engine.fcst_split, simple); mvd->set_raw(engine.fcst_raw, simple); mvd->set_shapedata(Fcst_sd, simple); @@ -2599,7 +2596,7 @@ void ModeExecutive::write_ct_stats() /////////////////////////////////////////////////////////////////////// -void ModeExecutive::conf_read(const string &default_config_filename) +void ModeExecutive::conf_read() { // Create the default config file name diff --git a/src/tools/core/mode/mode_exec.h b/src/tools/core/mode/mode_exec.h index 574bf937c6..f3c9937d35 100644 --- a/src/tools/core/mode/mode_exec.h +++ b/src/tools/core/mode/mode_exec.h @@ -82,56 +82,75 @@ class ModeExecutive { void init_traditional(int n_files); void init_multivar_simple(int j, int n_files, ModeDataType dtype, const ModeConfInfo &conf); - void init_multivar_intensities(GrdFileType ftype, GrdFileType otype, const ModeConfInfo &conf); - + void init_multivar_intensities(const ModeConfInfo &conf); int n_conv_radii () const; int n_conv_threshs () const; - int n_runs() const; + // these are used only for traditional mode int R_index; // indices into the convolution radius and threshold arrays int T_index; // for the current run // - // Input configuration files + // Input configuration files, all 3 used only for traditional mode + // Multivar mode handles the configs outside of the exec // - + // the hardwired default traditional mode config ConcatString default_config_file; - ConcatString match_config_file; + + // set for both trad and multivar, this is the default file on the command line + // used only for traditional + ConcatString match_config_file; + + // the extra one that can be set only for traditional mode ConcatString merge_config_file; // - // Input files + // Input filenames, set for both multivar and trad mode + // but used only for trad mode // - ConcatString fcst_file; ConcatString obs_file; + + // set and used only for trad mode Met2dDataFile * fcst_mtddf; Met2dDataFile * obs_mtddf; + // used for both trad and multivar mode TTContingencyTable cts[n_cts]; + // used for both trad and multivar mode ModeFuzzyEngine engine; // verification grid + // used for both trad and multivar mode, set by both Grid grid; Box xy_bb; ConcatString out_dir; + + // set for both trad and multivar mode, used for plotting limits double data_min, data_max; + // set for trad and multivar, used in the engine mode algorithm ShapeData Fcst_sd, Obs_sd; + // not used by multivar GrdFileType ftype, otype; + + // set into execs's conf varInfo object, only for multivar intensity comparisons + // for trad it's read in from the config string funits, ounits; + + // set into execs's conf varInfo object, only for multivar intensity comparisons + // for trad it's read in from the config string flevel, olevel; + // used in multivar only to customize outputs correctly bool isMultivarOutput; bool isMultivarSuperOutput; - //Processing_t ptype; - void setup_verification_grid(const ModeInputData &fcst, const ModeInputData &obs, const ModeConfInfo &conf); @@ -194,7 +213,9 @@ class ModeExecutive { void write_poly_netcdf(netCDF::NcFile *, const ObjPolyType); void write_ct_stats(); - void conf_read(const string &default_config_filename); + // traditional only, multivar reads outside of the exec + void conf_read(); + static string stype(Processing_t t); }; diff --git a/src/tools/core/mode/multivar_data.cc b/src/tools/core/mode/multivar_data.cc index 1d38d270ba..3cde9ccf83 100644 --- a/src/tools/core/mode/multivar_data.cc +++ b/src/tools/core/mode/multivar_data.cc @@ -142,8 +142,7 @@ MultiVarData::MultiVarData() : _merge(0), _name("notset"), _nx(0), _ny(0), - _grid(0), - _type(FileType_None) + _grid(0) { } @@ -152,23 +151,10 @@ MultiVarData::~MultiVarData() _clear(); } -void MultiVarData::checkFileTypeConsistency(const MultiVarData &mvdi, int j) -{ - bool err = false; - if (_type != mvdi._type) { - mlog << Error << "MultivarData::checkFileTypeConsistgency() -> " - << "inputs of different file types not supported " - << "Input 0:" << grdfiletype_to_string(_type).c_str() - << "Input " << j << ":" << grdfiletype_to_string(mvdi._type).c_str() - << "\n\n"; - exit ( 1 ); - } -} - void MultiVarData::init(ModeDataType dataType, const string &name, - const Grid &grid, GrdFileType type, - const string &units, + const Grid &grid, + const string &units, const string &level, double data_min, double data_max) { @@ -179,7 +165,6 @@ void MultiVarData::init(ModeDataType dataType, _nx = grid.nx(); _ny = grid.ny(); _grid = new Grid(grid); - _type = type; _units = units; _level = level; _data_min = data_min; @@ -286,7 +271,6 @@ void MultiVarData::_clear() _grid = 0; } _nx = _ny = 0; - _type = FileType_None; } diff --git a/src/tools/core/mode/multivar_data.h b/src/tools/core/mode/multivar_data.h index 142e1aa65b..f93b7f9ac5 100644 --- a/src/tools/core/mode/multivar_data.h +++ b/src/tools/core/mode/multivar_data.h @@ -85,13 +85,11 @@ class MultiVarData { MultiVarData(); ~MultiVarData(); - void checkFileTypeConsistency(const MultiVarData &mvdi, int j); - void init(ModeDataType dataType, const string &name, - const Grid &grid, GrdFileType type, + const Grid &grid, const string &units, - const string &level, + const string &level, double data_min, double data_max); void set_obj(ShapeData *sd, bool simple); @@ -109,7 +107,6 @@ class MultiVarData { string _name; int _nx, _ny; Grid *_grid; - GrdFileType _type; string _units; string _level; double _data_min, _data_max; diff --git a/src/tools/core/mode/multivar_frontend.cc b/src/tools/core/mode/multivar_frontend.cc index 4d88f37df5..6bd3a5aa57 100644 --- a/src/tools/core/mode/multivar_frontend.cc +++ b/src/tools/core/mode/multivar_frontend.cc @@ -103,9 +103,6 @@ int MultivarFrontEnd::run(const StringArray & Argv) MultiVarData *mvdi = create_simple_objects(ModeDataType_MvMode_Fcst, j, n_fcst_files, fcst_filenames[j], fcstInput[j]); - if (j > 0) { - mvdFcst[0]->checkFileTypeConsistency(*mvdi, j); - } mvdFcst.push_back(mvdi); mvdi->print(); } // for j @@ -118,9 +115,6 @@ int MultivarFrontEnd::run(const StringArray & Argv) MultiVarData *mvdi = create_simple_objects(ModeDataType_MvMode_Obs, j, n_obs_files, obs_filenames[j], obsInput[j]); - if (j > 0) { - mvdObs[0]->checkFileTypeConsistency(*mvdi, j); - } mvdObs.push_back(mvdi); mvdi->print(); } // for j @@ -281,11 +275,11 @@ void MultivarFrontEnd::read_input(const string &name, int index, ModeDataType ty if (type == ModeDataType_MvMode_Fcst) { config.process_config_field(ft, other_t, type, index); f->data_plane(*(config.Fcst->var_info), dp); - fcstInput.push_back(ModeInputData(name, dp, g, ft)); + fcstInput.push_back(ModeInputData(name, dp, g)); } else { config.process_config_field(other_t, ft, type, index); f->data_plane(*(config.Obs->var_info), dp); - obsInput.push_back(ModeInputData(name, dp, g, ft)); + obsInput.push_back(ModeInputData(name, dp, g)); } delete f; @@ -298,10 +292,6 @@ void MultivarFrontEnd::create_verif_grid() mlog << Debug(2) << "\n creating the verification grid \n" << sep << "\n"; _init_exec(ModeExecutive::TRADITIONAL, "None", "None"); - // mode_exec->init_multivar_verif_grid(fcstInput[0]._dataPlane, - // obsInput[0]._dataPlane, config); - // ModeConfInfo & conf = mode_exec->engine.conf_info; - // conf.set_field_index(0); mode_exec->setup_verification_grid(fcstInput[0], obsInput[0], config); verification_grid = mode_exec->grid; delete mode_exec; mode_exec = 0; @@ -350,7 +340,7 @@ MultivarFrontEnd::create_intensity_comparisons(int findex, int oindex, mlog << Debug(1) << "Running mvmode intensity comparisions \n\n"; _init_exec(ModeExecutive::MULTIVAR_INTENSITY, fcst_filename, obs_filename); - mode_exec->init_multivar_intensities(mvdf._type, mvdo._type, config); + mode_exec->init_multivar_intensities(config); ModeConfInfo & conf = mode_exec->engine.conf_info; conf.set_field_index(findex, oindex); @@ -400,7 +390,7 @@ void MultivarFrontEnd::process_superobjects(ModeSuperObject &fsuper, osuper.mask_data_super("ObsSimple", mvdo); _init_exec(ModeExecutive::MULTIVAR_SUPER, "None", "None"); - mode_exec->init_multivar_intensities(mvdf._type, mvdo._type, config); + mode_exec->init_multivar_intensities(config); ModeConfInfo & conf = mode_exec->engine.conf_info; if ((fsuper._hasUnion || osuper._hasUnion) && @@ -673,7 +663,8 @@ void MultivarFrontEnd::_init_exec(ModeExecutive::Processing_t p, // compress_level = -1; mode_exec->fcst_file = ffile; mode_exec->obs_file = ofile; - mode_exec->match_config_file = config_file; + + mode_exec->match_config_file = config_file; // this is never used mode_exec->out_dir = output_path; } From 5694a2aa8dc7062cffab2020505ddb4d59fed913 Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Tue, 2 Jan 2024 23:14:36 +0000 Subject: [PATCH 05/14] Bug fix to doc --- docs/Users_Guide/mode.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/Users_Guide/mode.rst b/docs/Users_Guide/mode.rst index becf20ed98..b5fa96f258 100644 --- a/docs/Users_Guide/mode.rst +++ b/docs/Users_Guide/mode.rst @@ -131,13 +131,13 @@ When setting a threshold to a percentile, some choices require both an observati * "==FBIAS" in an observation input. * e.g. "==FBIAS1" in an observation input to automatically de-bias the data, using a simple threshold in the matching forecast input. For example, - when observation input 3 has "==FBIAS1", and forecast input 3 has ">5.0", MET applies the >5.0 threshold to the forecast and then chooses an - observation threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0. + when observation input 3 has "==FBIAS1", and forecast input 3 has ">5.0", MET applies the >5.0 threshold to the forecast and then chooses an + observation threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0. * "==FBIAS" in a forecast input. * e.g. "==FBIAS1" in a forecast input to automatically de-bias the data, using a simple threshold in the matching observation input. For example, - when forecast input 2 has "==FBIAS1", and observation input 2 has ">5.0", MET applies the >5.0 threshold to the observation and then chooses a - forecast threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0. + when forecast input 2 has "==FBIAS1", and observation input 2 has ">5.0", MET applies the >5.0 threshold to the observation and then chooses a + forecast threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0. Practical Information From 72d39fa5003023f16d728a9e995b1dc1154727ce Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Tue, 2 Jan 2024 23:22:19 +0000 Subject: [PATCH 06/14] Another bugfix attempt --- docs/Users_Guide/mode.rst | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/docs/Users_Guide/mode.rst b/docs/Users_Guide/mode.rst index b5fa96f258..0e5c5bdbcb 100644 --- a/docs/Users_Guide/mode.rst +++ b/docs/Users_Guide/mode.rst @@ -123,21 +123,16 @@ When regridding to the FCST or OBS field (e.g. to_grid = FCST), the first field When setting a threshold to a percentile, some choices require both an observation input and a forecast input. When this is the case, it's assumed the indices match, so for example if forecast input 1 has such a percentile setting, then observation input 1 will be used to compute the percentile. Percentiles in which this will happen are: * SFP in an observation input - * The matching forecast input will be used to determine the threshold. - e.g. ">SFP33.3" in the 2nd observation input means greater than 33.3-rd percentile of the 2nd forecast input will be used as the threshold for that observation input. + * The matching forecast input will be used to determine the threshold. e.g. ">SFP33.3" in the 2nd observation input means greater than 33.3-rd percentile of the 2nd forecast input will be used as the threshold for that observation input. + * SOP in a forecast input - * The matching observation input will be used to determine the threshold. - e.g. ">SOP33.3" in the 2nd forecast input means greater than 33.3-rd percentile of the 2nd observation input will be used as the threshold for that forecast input. + * The matching observation input will be used to determine the threshold. e.g. ">SOP33.3" in the 2nd forecast input means greater than 33.3-rd percentile of the 2nd observation input will be used as the threshold for that forecast input. * "==FBIAS" in an observation input. - * e.g. "==FBIAS1" in an observation input to automatically de-bias the data, using a simple threshold in the matching forecast input. For example, - when observation input 3 has "==FBIAS1", and forecast input 3 has ">5.0", MET applies the >5.0 threshold to the forecast and then chooses an - observation threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0. + * e.g. "==FBIAS1" in an observation input to automatically de-bias the data, using a simple threshold in the matching forecast input. For example, when observation input 3 has "==FBIAS1", and forecast input 3 has ">5.0", MET applies the >5.0 threshold to the forecast and then chooses an observation threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0. * "==FBIAS" in a forecast input. - * e.g. "==FBIAS1" in a forecast input to automatically de-bias the data, using a simple threshold in the matching observation input. For example, - when forecast input 2 has "==FBIAS1", and observation input 2 has ">5.0", MET applies the >5.0 threshold to the observation and then chooses a - forecast threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0. + * e.g. "==FBIAS1" in a forecast input to automatically de-bias the data, using a simple threshold in the matching observation input. For example, when forecast input 2 has "==FBIAS1", and observation input 2 has ">5.0", MET applies the >5.0 threshold to the observation and then chooses a forecast threshold which results in a frequency bias of 1. The frequency bias can be any float value > 0.0. Practical Information From cba321a254246c5aa47af26f5d79be06258b9165 Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Wed, 3 Jan 2024 23:04:18 +0000 Subject: [PATCH 07/14] synching up development.seneca with what is in the develop branch --- internal/scripts/environment/development.seneca | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/scripts/environment/development.seneca b/internal/scripts/environment/development.seneca index 5c69a32e9e..928cee18db 100644 --- a/internal/scripts/environment/development.seneca +++ b/internal/scripts/environment/development.seneca @@ -48,6 +48,9 @@ export MET_FONT_DIR=${MET_TEST_INPUT}/fonts # Define Rscript to use a version with the ncdf4 package 1.17 or later export MET_TEST_RSCRIPT=/nrit/ral/R-4.3.1/bin/Rscript +# Define runtime Python version +export MET_TEST_MET_PYTHON_EXE=${MET_PYTHON_BIN_EXE} + # This is a cron script -- create the shell environment for this job # - NCO is for ncdiff # - NetCDF is for ncdump. From 07e45effa20d6d3c30764cdbe1e2267fe877f6ee Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Thu, 4 Jan 2024 22:33:58 +0000 Subject: [PATCH 08/14] added clone() method to all the vx_data2d var_info classes --- src/libcode/vx_data2d/var_info.h | 5 ++++- src/libcode/vx_data2d_grib/var_info_grib.cc | 8 ++++++++ src/libcode/vx_data2d_grib/var_info_grib.h | 1 + src/libcode/vx_data2d_grib2/var_info_grib2.cc | 8 ++++++++ src/libcode/vx_data2d_grib2/var_info_grib2.h | 1 + src/libcode/vx_data2d_nc_met/var_info_nc_met.cc | 7 +++++++ src/libcode/vx_data2d_nc_met/var_info_nc_met.h | 1 + .../vx_data2d_nc_pinterp/var_info_nc_pinterp.cc | 8 ++++++++ .../vx_data2d_nc_pinterp/var_info_nc_pinterp.h | 2 ++ src/libcode/vx_data2d_nccf/var_info_nccf.cc | 8 ++++++++ src/libcode/vx_data2d_nccf/var_info_nccf.h | 1 + src/libcode/vx_data2d_python/var_info_python.cc | 8 ++++++++ src/libcode/vx_data2d_python/var_info_python.h | 1 + src/libcode/vx_data2d_ugrid/var_info_ugrid.cc | 8 ++++++++ src/libcode/vx_data2d_ugrid/var_info_ugrid.h | 1 + src/libcode/vx_shapedata/mode_field_info.cc | 12 +++--------- 16 files changed, 70 insertions(+), 10 deletions(-) diff --git a/src/libcode/vx_data2d/var_info.h b/src/libcode/vx_data2d/var_info.h index 7dcf6e386a..15fa2f90b7 100644 --- a/src/libcode/vx_data2d/var_info.h +++ b/src/libcode/vx_data2d/var_info.h @@ -91,11 +91,14 @@ class VarInfo VarInfo(const VarInfo &); VarInfo & operator=(const VarInfo &); + virtual VarInfo *clone() const = 0; + // Conversion function UserFunc_1Arg ConvertFx; void clear(); - + void clone_base() const; + virtual void dump(std::ostream &) const; // diff --git a/src/libcode/vx_data2d_grib/var_info_grib.cc b/src/libcode/vx_data2d_grib/var_info_grib.cc index c414bc94f8..cfe81f5a4a 100644 --- a/src/libcode/vx_data2d_grib/var_info_grib.cc +++ b/src/libcode/vx_data2d_grib/var_info_grib.cc @@ -73,6 +73,14 @@ VarInfoGrib & VarInfoGrib::operator=(const VarInfoGrib &f) { return ( *this ); } +/////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoGrib::clone() const +{ + VarInfoGrib *ret = new VarInfoGrib(*this); + return (VarInfo *)ret; +} + + /////////////////////////////////////////////////////////////////////////////// void VarInfoGrib::init_from_scratch() { diff --git a/src/libcode/vx_data2d_grib/var_info_grib.h b/src/libcode/vx_data2d_grib/var_info_grib.h index cdf0947b90..d695eed007 100644 --- a/src/libcode/vx_data2d_grib/var_info_grib.h +++ b/src/libcode/vx_data2d_grib/var_info_grib.h @@ -67,6 +67,7 @@ class VarInfoGrib : public VarInfo ~VarInfoGrib(); VarInfoGrib(const VarInfoGrib &); VarInfoGrib & operator=(const VarInfoGrib &); + VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_data2d_grib2/var_info_grib2.cc b/src/libcode/vx_data2d_grib2/var_info_grib2.cc index 90f5275bbc..1d30065c35 100644 --- a/src/libcode/vx_data2d_grib2/var_info_grib2.cc +++ b/src/libcode/vx_data2d_grib2/var_info_grib2.cc @@ -77,6 +77,14 @@ VarInfoGrib2 & VarInfoGrib2::operator=(const VarInfoGrib2 &f) { /////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoGrib2::clone() const +{ + VarInfoGrib2 *ret = new VarInfoGrib2(*this); + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoGrib2::init_from_scratch() { // First call the parent's initialization diff --git a/src/libcode/vx_data2d_grib2/var_info_grib2.h b/src/libcode/vx_data2d_grib2/var_info_grib2.h index 7f47facf08..ef6502ce35 100644 --- a/src/libcode/vx_data2d_grib2/var_info_grib2.h +++ b/src/libcode/vx_data2d_grib2/var_info_grib2.h @@ -67,6 +67,7 @@ class VarInfoGrib2 : public VarInfo ~VarInfoGrib2(); VarInfoGrib2(const VarInfoGrib2 &); VarInfoGrib2 & operator=(const VarInfoGrib2 &); + VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_data2d_nc_met/var_info_nc_met.cc b/src/libcode/vx_data2d_nc_met/var_info_nc_met.cc index 651832d69d..dcfbdd9da2 100644 --- a/src/libcode/vx_data2d_nc_met/var_info_nc_met.cc +++ b/src/libcode/vx_data2d_nc_met/var_info_nc_met.cc @@ -74,6 +74,13 @@ VarInfoNcMet & VarInfoNcMet::operator=(const VarInfoNcMet &f) { return ( *this ); } +/////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoNcMet::clone() const +{ + VarInfoNcMet *ret = new VarInfoNcMet(*this); + return (VarInfo *)ret; +} + /////////////////////////////////////////////////////////////////////////////// void VarInfoNcMet::init_from_scratch() { diff --git a/src/libcode/vx_data2d_nc_met/var_info_nc_met.h b/src/libcode/vx_data2d_nc_met/var_info_nc_met.h index a9f0c66f53..f85bdfdb7d 100644 --- a/src/libcode/vx_data2d_nc_met/var_info_nc_met.h +++ b/src/libcode/vx_data2d_nc_met/var_info_nc_met.h @@ -41,6 +41,7 @@ class VarInfoNcMet : public VarInfo ~VarInfoNcMet(); VarInfoNcMet(const VarInfoNcMet &); VarInfoNcMet & operator=(const VarInfoNcMet &); + virtual VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_data2d_nc_pinterp/var_info_nc_pinterp.cc b/src/libcode/vx_data2d_nc_pinterp/var_info_nc_pinterp.cc index f0e01dc6a6..2bdf449143 100644 --- a/src/libcode/vx_data2d_nc_pinterp/var_info_nc_pinterp.cc +++ b/src/libcode/vx_data2d_nc_pinterp/var_info_nc_pinterp.cc @@ -71,6 +71,14 @@ VarInfoNcPinterp & VarInfoNcPinterp::operator=(const VarInfoNcPinterp &f) { /////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoNcPinterp::clone() const +{ + VarInfoNcPinterp *ret = new VarInfoNcPinterp(*this); + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoNcPinterp::init_from_scratch() { // First call the parent's initialization diff --git a/src/libcode/vx_data2d_nc_pinterp/var_info_nc_pinterp.h b/src/libcode/vx_data2d_nc_pinterp/var_info_nc_pinterp.h index 61cc0204c0..9ea1cf6322 100644 --- a/src/libcode/vx_data2d_nc_pinterp/var_info_nc_pinterp.h +++ b/src/libcode/vx_data2d_nc_pinterp/var_info_nc_pinterp.h @@ -200,6 +200,8 @@ class VarInfoNcPinterp : public VarInfo ~VarInfoNcPinterp(); VarInfoNcPinterp(const VarInfoNcPinterp &); VarInfoNcPinterp & operator=(const VarInfoNcPinterp &); + VarInfo *clone() const; + void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_data2d_nccf/var_info_nccf.cc b/src/libcode/vx_data2d_nccf/var_info_nccf.cc index a77ece6881..2a9b1e83df 100644 --- a/src/libcode/vx_data2d_nccf/var_info_nccf.cc +++ b/src/libcode/vx_data2d_nccf/var_info_nccf.cc @@ -76,6 +76,14 @@ VarInfoNcCF & VarInfoNcCF::operator=(const VarInfoNcCF &f) { /////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoNcCF::clone() const +{ + VarInfoNcCF *ret = new VarInfoNcCF(*this); + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoNcCF::init_from_scratch() { // First call the parent's initialization diff --git a/src/libcode/vx_data2d_nccf/var_info_nccf.h b/src/libcode/vx_data2d_nccf/var_info_nccf.h index 2463bcbde2..ce20289c2c 100644 --- a/src/libcode/vx_data2d_nccf/var_info_nccf.h +++ b/src/libcode/vx_data2d_nccf/var_info_nccf.h @@ -45,6 +45,7 @@ class VarInfoNcCF : public VarInfo ~VarInfoNcCF(); VarInfoNcCF(const VarInfoNcCF &); VarInfoNcCF & operator=(const VarInfoNcCF &); + VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_data2d_python/var_info_python.cc b/src/libcode/vx_data2d_python/var_info_python.cc index 5c111ec28b..5454c58454 100644 --- a/src/libcode/vx_data2d_python/var_info_python.cc +++ b/src/libcode/vx_data2d_python/var_info_python.cc @@ -75,6 +75,14 @@ VarInfoPython & VarInfoPython::operator=(const VarInfoPython &f) { /////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoPython::clone() const +{ + VarInfoPython *ret = new VarInfoPython(*this); + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoPython::init_from_scratch() { // First call the parent's initialization diff --git a/src/libcode/vx_data2d_python/var_info_python.h b/src/libcode/vx_data2d_python/var_info_python.h index 68309f8b4a..e38f9edae2 100644 --- a/src/libcode/vx_data2d_python/var_info_python.h +++ b/src/libcode/vx_data2d_python/var_info_python.h @@ -41,6 +41,7 @@ class VarInfoPython : public VarInfo ~VarInfoPython(); VarInfoPython(const VarInfoPython &); VarInfoPython & operator=(const VarInfoPython &); + VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_data2d_ugrid/var_info_ugrid.cc b/src/libcode/vx_data2d_ugrid/var_info_ugrid.cc index a40844de76..83da12251c 100644 --- a/src/libcode/vx_data2d_ugrid/var_info_ugrid.cc +++ b/src/libcode/vx_data2d_ugrid/var_info_ugrid.cc @@ -77,6 +77,14 @@ VarInfoUGrid & VarInfoUGrid::operator=(const VarInfoUGrid &f) { /////////////////////////////////////////////////////////////////////////////// +VarInfo *VarInfoUGrid::clone() const +{ + VarInfoUGrid *ret = new VarInfoUGrid(*this); + return (VarInfo *)ret; +} + +/////////////////////////////////////////////////////////////////////////////// + void VarInfoUGrid::init_from_scratch() { // First call the parent's initialization diff --git a/src/libcode/vx_data2d_ugrid/var_info_ugrid.h b/src/libcode/vx_data2d_ugrid/var_info_ugrid.h index b7c64b1014..e3bb5a1bb9 100644 --- a/src/libcode/vx_data2d_ugrid/var_info_ugrid.h +++ b/src/libcode/vx_data2d_ugrid/var_info_ugrid.h @@ -45,6 +45,7 @@ class VarInfoUGrid : public VarInfo ~VarInfoUGrid(); VarInfoUGrid(const VarInfoUGrid &); VarInfoUGrid & operator=(const VarInfoUGrid &); + VarInfo *clone() const; void dump(std::ostream &) const; void clear(); diff --git a/src/libcode/vx_shapedata/mode_field_info.cc b/src/libcode/vx_shapedata/mode_field_info.cc index 9569dac765..fcd652a09a 100644 --- a/src/libcode/vx_shapedata/mode_field_info.cc +++ b/src/libcode/vx_shapedata/mode_field_info.cc @@ -97,13 +97,9 @@ void Mode_Field_Info::clone(const Mode_Field_Info & i) index = i.index; conv_radius = i.conv_radius; vld_thresh = i.vld_thresh; - VarInfoFactory info_factory; - var_info = info_factory.new_var_info(gft); - if ( Multivar ) { - var_info->set_dict(*dict); - } else { - var_info->set_dict(*(dict->lookup_dictionary(conf_key_field))); - } + + var_info = i.var_info->clone(); + conv_radius_array = i.conv_radius_array; conv_thresh_array = i.conv_thresh_array; merge_thresh_array = i.merge_thresh_array; @@ -113,8 +109,6 @@ void Mode_Field_Info::clone(const Mode_Field_Info & i) raw_pi = i.raw_pi; filter_attr_map = i.filter_attr_map; file_type = i.file_type; - - } //////////////////////////////////////////////////////////////////////// From 0fdd088decf9f9f659c4474fe6513e4556a120b4 Mon Sep 17 00:00:00 2001 From: Tracy Hertneky Date: Fri, 12 Jan 2024 20:51:31 +0000 Subject: [PATCH 09/14] Fix multivar_name/level definitions in UG --- docs/Users_Guide/mode.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/Users_Guide/mode.rst b/docs/Users_Guide/mode.rst index 0e5c5bdbcb..4f7d7e3464 100644 --- a/docs/Users_Guide/mode.rst +++ b/docs/Users_Guide/mode.rst @@ -277,7 +277,7 @@ _____________________ multivar_name = "Super"; -The **multivar_name** entry appears only in the **MODEMultivarConfig_default** file. This option is used only when the multivar option is enabled, and only when all **multivar_intensity_flag** values are FALSE. It can be thought of as an identifier for the multivariate super object. It shows up in output files names and content. It can be set separately for forecasts and observations or as a common value for both. +The **multivar_name** entry appears only in the **MODEMultivarConfig_default** file. This option is used only when the multivar option is enabled, and only when **multivar_intensity_compare_fcst** and **multivar_intensity_compare_obs** are empty. It can be thought of as an identifier for the multivariate super object. It shows up in output files names and content. It can be set separately for forecasts and observations or as a common value for both. _____________________ @@ -285,7 +285,7 @@ _____________________ multivar_level = "LO"; -The **multivar_level** entry appears only in the **MODEMultivarConfig_default** file. This option is used only when the multivar option is enabled, and only when all **multivar_intensity_flag** values are FALSE. It is the identifier for the multivariate super object as regards level. It shows up in output files names and content. If not set the default value is "NA". It can be set separately for forecasts and observations, or as a common value for both. +The **multivar_level** entry appears only in the **MODEMultivarConfig_default** file. This option is used only when the multivar option is enabled, and only when **multivar_intensity_compare_fcst** and **multivar_intensity_compare_obs** are empty. It is the identifier for the multivariate super object as regards level. It shows up in output files names and content. If not set the default value is "NA". It can be set separately for forecasts and observations, or as a common value for both. _____________________ From 84d8990a83a9907d75da195010ca1f45bf6e7c6e Mon Sep 17 00:00:00 2001 From: Dave Albo Date: Tue, 16 Jan 2024 19:41:56 +0000 Subject: [PATCH 10/14] bug fix regarding accessing the config to create a verification grid --- src/tools/core/mode/mode_exec.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tools/core/mode/mode_exec.cc b/src/tools/core/mode/mode_exec.cc index c3dc975fda..3a2ce45538 100644 --- a/src/tools/core/mode/mode_exec.cc +++ b/src/tools/core/mode/mode_exec.cc @@ -414,6 +414,10 @@ void ModeExecutive::setup_verification_grid(const ModeInputData &fcst, Fcst_sd.data = fcst._dataPlane; Obs_sd.data = obs._dataPlane; + // set this local conf to point to forecast 0, so that that regrid info + // can be accessed and it can be decided if we are to use fcst or obs + // input + engine.conf_info.set_data_type(ModeDataType_MvMode_Fcst); engine.conf_info.set_field_index(0); grid = parse_vx_grid(engine.conf_info.Fcst->var_info->regrid(), &fcst._grid, &obs._grid); From 960f8e695c6445394f86f4903aea643cf6028411 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 29 Jan 2024 13:43:28 -0700 Subject: [PATCH 11/14] Update docs/Users_Guide/mode.rst --- docs/Users_Guide/mode.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Users_Guide/mode.rst b/docs/Users_Guide/mode.rst index 4f7d7e3464..7fc1a162f3 100644 --- a/docs/Users_Guide/mode.rst +++ b/docs/Users_Guide/mode.rst @@ -122,7 +122,7 @@ When regridding to the FCST or OBS field (e.g. to_grid = FCST), the first field When setting a threshold to a percentile, some choices require both an observation input and a forecast input. When this is the case, it's assumed the indices match, so for example if forecast input 1 has such a percentile setting, then observation input 1 will be used to compute the percentile. Percentiles in which this will happen are: -* SFP in an observation input +* SFP in an observation input. * The matching forecast input will be used to determine the threshold. e.g. ">SFP33.3" in the 2nd observation input means greater than 33.3-rd percentile of the 2nd forecast input will be used as the threshold for that observation input. * SOP in a forecast input From 005bb1155ba0e15852c703822c5413df782adc67 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Mon, 29 Jan 2024 13:43:38 -0700 Subject: [PATCH 12/14] Update docs/Users_Guide/mode.rst --- docs/Users_Guide/mode.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Users_Guide/mode.rst b/docs/Users_Guide/mode.rst index 7fc1a162f3..491b452002 100644 --- a/docs/Users_Guide/mode.rst +++ b/docs/Users_Guide/mode.rst @@ -125,7 +125,7 @@ When setting a threshold to a percentile, some choices require both an observati * SFP in an observation input. * The matching forecast input will be used to determine the threshold. e.g. ">SFP33.3" in the 2nd observation input means greater than 33.3-rd percentile of the 2nd forecast input will be used as the threshold for that observation input. -* SOP in a forecast input +* SOP in a forecast input. * The matching observation input will be used to determine the threshold. e.g. ">SOP33.3" in the 2nd forecast input means greater than 33.3-rd percentile of the 2nd observation input will be used as the threshold for that forecast input. * "==FBIAS" in an observation input. From c34eb029abcb1b921ea4c1c43b6107d5502dfe64 Mon Sep 17 00:00:00 2001 From: MET Tools Test Account Date: Mon, 29 Jan 2024 21:13:01 +0000 Subject: [PATCH 13/14] Per #2745, commit changes to Makefile.am/.in files after running bootstrap on this feature branch on seneca. --- src/libcode/vx_shapedata/Makefile.am | 2 +- src/libcode/vx_shapedata/Makefile.in | 1 + src/tools/core/mode/Makefile.in | 25 ++++++++++++++++++++++--- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/libcode/vx_shapedata/Makefile.am b/src/libcode/vx_shapedata/Makefile.am index b5fe46ed68..ea52307df3 100644 --- a/src/libcode/vx_shapedata/Makefile.am +++ b/src/libcode/vx_shapedata/Makefile.am @@ -21,7 +21,7 @@ libvx_shapedata_a_SOURCES = \ mode_columns.h \ mode_field_info.cc mode_field_info.h \ mode_conf_info.cc mode_conf_info.h \ - mode_input_data.h \ + mode_input_data.h \ engine.cc engine.h \ ihull.cc ihull.h \ vx_shapedata.h diff --git a/src/libcode/vx_shapedata/Makefile.in b/src/libcode/vx_shapedata/Makefile.in index df218fceda..290ee7582f 100644 --- a/src/libcode/vx_shapedata/Makefile.in +++ b/src/libcode/vx_shapedata/Makefile.in @@ -369,6 +369,7 @@ libvx_shapedata_a_SOURCES = \ mode_columns.h \ mode_field_info.cc mode_field_info.h \ mode_conf_info.cc mode_conf_info.h \ + mode_input_data.h \ engine.cc engine.h \ ihull.cc ihull.h \ vx_shapedata.h diff --git a/src/tools/core/mode/Makefile.in b/src/tools/core/mode/Makefile.in index 741c5fea4f..10f09a3200 100644 --- a/src/tools/core/mode/Makefile.in +++ b/src/tools/core/mode/Makefile.in @@ -102,9 +102,9 @@ CONFIG_CLEAN_VPATH_FILES = am__installdirs = "$(DESTDIR)$(bindir)" PROGRAMS = $(bin_PROGRAMS) am_mode_OBJECTS = mode-mode_usage.$(OBJEXT) \ - mode-mode_frontend.$(OBJEXT) mode-multivar_data.$(OBJEXT) \ - mode-multivar_frontend.$(OBJEXT) mode-mode.$(OBJEXT) \ - mode-combine_boolplanes.$(OBJEXT) \ + mode-mode_frontend.$(OBJEXT) mode-mode_superobject.$(OBJEXT) \ + mode-multivar_data.$(OBJEXT) mode-multivar_frontend.$(OBJEXT) \ + mode-mode.$(OBJEXT) mode-combine_boolplanes.$(OBJEXT) \ mode-objects_from_netcdf.$(OBJEXT) mode-mode_ps_file.$(OBJEXT) \ mode-plot_engine.$(OBJEXT) mode-page_1.$(OBJEXT) \ mode-fcst_enlarge_page.$(OBJEXT) \ @@ -138,6 +138,7 @@ am__depfiles_remade = ./$(DEPDIR)/mode-cluster_page.Po \ ./$(DEPDIR)/mode-mode_exec.Po \ ./$(DEPDIR)/mode-mode_frontend.Po \ ./$(DEPDIR)/mode-mode_ps_file.Po \ + ./$(DEPDIR)/mode-mode_superobject.Po \ ./$(DEPDIR)/mode-mode_usage.Po \ ./$(DEPDIR)/mode-multivar_data.Po \ ./$(DEPDIR)/mode-multivar_frontend.Po \ @@ -350,6 +351,7 @@ top_srcdir = @top_srcdir@ MAINTAINERCLEANFILES = Makefile.in mode_SOURCES = mode_usage.cc \ mode_frontend.cc \ + mode_superobject.cc \ multivar_data.cc \ multivar_frontend.cc \ mode.cc \ @@ -503,6 +505,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-mode_exec.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-mode_frontend.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-mode_ps_file.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-mode_superobject.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-mode_usage.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-multivar_data.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/mode-multivar_frontend.Po@am__quote@ # am--include-marker @@ -560,6 +563,20 @@ mode-mode_frontend.obj: mode_frontend.cc @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(mode_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o mode-mode_frontend.obj `if test -f 'mode_frontend.cc'; then $(CYGPATH_W) 'mode_frontend.cc'; else $(CYGPATH_W) '$(srcdir)/mode_frontend.cc'; fi` +mode-mode_superobject.o: mode_superobject.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(mode_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT mode-mode_superobject.o -MD -MP -MF $(DEPDIR)/mode-mode_superobject.Tpo -c -o mode-mode_superobject.o `test -f 'mode_superobject.cc' || echo '$(srcdir)/'`mode_superobject.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/mode-mode_superobject.Tpo $(DEPDIR)/mode-mode_superobject.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mode_superobject.cc' object='mode-mode_superobject.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(mode_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o mode-mode_superobject.o `test -f 'mode_superobject.cc' || echo '$(srcdir)/'`mode_superobject.cc + +mode-mode_superobject.obj: mode_superobject.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(mode_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT mode-mode_superobject.obj -MD -MP -MF $(DEPDIR)/mode-mode_superobject.Tpo -c -o mode-mode_superobject.obj `if test -f 'mode_superobject.cc'; then $(CYGPATH_W) 'mode_superobject.cc'; else $(CYGPATH_W) '$(srcdir)/mode_superobject.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/mode-mode_superobject.Tpo $(DEPDIR)/mode-mode_superobject.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='mode_superobject.cc' object='mode-mode_superobject.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(mode_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o mode-mode_superobject.obj `if test -f 'mode_superobject.cc'; then $(CYGPATH_W) 'mode_superobject.cc'; else $(CYGPATH_W) '$(srcdir)/mode_superobject.cc'; fi` + mode-multivar_data.o: multivar_data.cc @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(mode_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT mode-multivar_data.o -MD -MP -MF $(DEPDIR)/mode-multivar_data.Tpo -c -o mode-multivar_data.o `test -f 'multivar_data.cc' || echo '$(srcdir)/'`multivar_data.cc @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/mode-multivar_data.Tpo $(DEPDIR)/mode-multivar_data.Po @@ -876,6 +893,7 @@ distclean: distclean-am -rm -f ./$(DEPDIR)/mode-mode_exec.Po -rm -f ./$(DEPDIR)/mode-mode_frontend.Po -rm -f ./$(DEPDIR)/mode-mode_ps_file.Po + -rm -f ./$(DEPDIR)/mode-mode_superobject.Po -rm -f ./$(DEPDIR)/mode-mode_usage.Po -rm -f ./$(DEPDIR)/mode-multivar_data.Po -rm -f ./$(DEPDIR)/mode-multivar_frontend.Po @@ -936,6 +954,7 @@ maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/mode-mode_exec.Po -rm -f ./$(DEPDIR)/mode-mode_frontend.Po -rm -f ./$(DEPDIR)/mode-mode_ps_file.Po + -rm -f ./$(DEPDIR)/mode-mode_superobject.Po -rm -f ./$(DEPDIR)/mode-mode_usage.Po -rm -f ./$(DEPDIR)/mode-multivar_data.Po -rm -f ./$(DEPDIR)/mode-multivar_frontend.Po From a6dfa66867131579620787502fd1371f8262d046 Mon Sep 17 00:00:00 2001 From: MET Tools Test Account Date: Mon, 29 Jan 2024 21:57:26 +0000 Subject: [PATCH 14/14] Per #2745, no real code changes. Just whitespace. --- src/libcode/vx_data2d_grib/var_info_grib.cc | 7 ++++--- src/libcode/vx_data2d_grib2/var_info_grib2.cc | 5 +++-- src/libcode/vx_data2d_nc_cf/var_info_nc_cf.cc | 5 +++-- src/libcode/vx_data2d_nc_met/var_info_nc_met.cc | 6 ++++-- src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.cc | 1 + src/libcode/vx_data2d_python/var_info_python.cc | 5 +++-- src/libcode/vx_data2d_ugrid/var_info_ugrid.cc | 6 +++--- src/libcode/vx_shapedata/engine.cc | 5 ++--- src/libcode/vx_shapedata/engine.h | 1 + 9 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/libcode/vx_data2d_grib/var_info_grib.cc b/src/libcode/vx_data2d_grib/var_info_grib.cc index cfe81f5a4a..e6bb74198f 100644 --- a/src/libcode/vx_data2d_grib/var_info_grib.cc +++ b/src/libcode/vx_data2d_grib/var_info_grib.cc @@ -74,13 +74,14 @@ VarInfoGrib & VarInfoGrib::operator=(const VarInfoGrib &f) { } /////////////////////////////////////////////////////////////////////////////// -VarInfo *VarInfoGrib::clone() const -{ + +VarInfo *VarInfoGrib::clone() const { + VarInfoGrib *ret = new VarInfoGrib(*this); + return (VarInfo *)ret; } - /////////////////////////////////////////////////////////////////////////////// void VarInfoGrib::init_from_scratch() { diff --git a/src/libcode/vx_data2d_grib2/var_info_grib2.cc b/src/libcode/vx_data2d_grib2/var_info_grib2.cc index ab159b3407..ef3a3c7db6 100644 --- a/src/libcode/vx_data2d_grib2/var_info_grib2.cc +++ b/src/libcode/vx_data2d_grib2/var_info_grib2.cc @@ -77,9 +77,10 @@ VarInfoGrib2 & VarInfoGrib2::operator=(const VarInfoGrib2 &f) { /////////////////////////////////////////////////////////////////////////////// -VarInfo *VarInfoGrib2::clone() const -{ +VarInfo *VarInfoGrib2::clone() const { + VarInfoGrib2 *ret = new VarInfoGrib2(*this); + return (VarInfo *)ret; } diff --git a/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.cc b/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.cc index a1a4e6eba0..dba8bd4ec5 100644 --- a/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.cc +++ b/src/libcode/vx_data2d_nc_cf/var_info_nc_cf.cc @@ -76,9 +76,10 @@ VarInfoNcCF & VarInfoNcCF::operator=(const VarInfoNcCF &f) { /////////////////////////////////////////////////////////////////////////////// -VarInfo *VarInfoNcCF::clone() const -{ +VarInfo *VarInfoNcCF::clone() const { + VarInfoNcCF *ret = new VarInfoNcCF(*this); + return (VarInfo *)ret; } diff --git a/src/libcode/vx_data2d_nc_met/var_info_nc_met.cc b/src/libcode/vx_data2d_nc_met/var_info_nc_met.cc index dcfbdd9da2..c1624d8785 100644 --- a/src/libcode/vx_data2d_nc_met/var_info_nc_met.cc +++ b/src/libcode/vx_data2d_nc_met/var_info_nc_met.cc @@ -75,9 +75,11 @@ VarInfoNcMet & VarInfoNcMet::operator=(const VarInfoNcMet &f) { } /////////////////////////////////////////////////////////////////////////////// -VarInfo *VarInfoNcMet::clone() const -{ + +VarInfo *VarInfoNcMet::clone() const { + VarInfoNcMet *ret = new VarInfoNcMet(*this); + return (VarInfo *)ret; } diff --git a/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.cc b/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.cc index e2d12532f4..351ddf302a 100644 --- a/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.cc +++ b/src/libcode/vx_data2d_nc_wrf/var_info_nc_wrf.cc @@ -75,6 +75,7 @@ VarInfoNcWrf & VarInfoNcWrf::operator=(const VarInfoNcWrf &f) { VarInfo *VarInfoNcWrf::clone() const { VarInfoNcWrf *ret = new VarInfoNcWrf(*this); + return (VarInfo *)ret; } diff --git a/src/libcode/vx_data2d_python/var_info_python.cc b/src/libcode/vx_data2d_python/var_info_python.cc index 5454c58454..74d8767816 100644 --- a/src/libcode/vx_data2d_python/var_info_python.cc +++ b/src/libcode/vx_data2d_python/var_info_python.cc @@ -75,9 +75,10 @@ VarInfoPython & VarInfoPython::operator=(const VarInfoPython &f) { /////////////////////////////////////////////////////////////////////////////// -VarInfo *VarInfoPython::clone() const -{ +VarInfo *VarInfoPython::clone() const { + VarInfoPython *ret = new VarInfoPython(*this); + return (VarInfo *)ret; } diff --git a/src/libcode/vx_data2d_ugrid/var_info_ugrid.cc b/src/libcode/vx_data2d_ugrid/var_info_ugrid.cc index 2fdf681b2e..9e74762a48 100644 --- a/src/libcode/vx_data2d_ugrid/var_info_ugrid.cc +++ b/src/libcode/vx_data2d_ugrid/var_info_ugrid.cc @@ -32,7 +32,6 @@ using namespace std; - /////////////////////////////////////////////////////////////////////////////// static bool is_grib_code_abbr_match(const ConcatString &, int); @@ -77,9 +76,10 @@ VarInfoUGrid & VarInfoUGrid::operator=(const VarInfoUGrid &f) { /////////////////////////////////////////////////////////////////////////////// -VarInfo *VarInfoUGrid::clone() const -{ +VarInfo *VarInfoUGrid::clone() const { + VarInfoUGrid *ret = new VarInfoUGrid(*this); + return (VarInfo *)ret; } diff --git a/src/libcode/vx_shapedata/engine.cc b/src/libcode/vx_shapedata/engine.cc index c5bcdf9b46..6087bf62bd 100644 --- a/src/libcode/vx_shapedata/engine.cc +++ b/src/libcode/vx_shapedata/engine.cc @@ -1004,14 +1004,13 @@ void ModeFuzzyEngine::do_obs_merging(const ShapeData &merge_data) if(need_obs_thresh) do_obs_thresholding(); if(!need_obs_merge) return; - + if (obs_thresh->data.nx() != merge_data.data.nx() || obs_thresh->data.ny() != merge_data.data.ny()) { mlog << Error << "\nModeFuzzyEngine::do_obs_merging() -> " << "inconsistent array dims\n\n"; exit(1); } - if(conf_info.Obs->merge_flag == MergeType_Both || conf_info.Obs->merge_flag == MergeType_Thresh) @@ -1020,7 +1019,7 @@ void ModeFuzzyEngine::do_obs_merging(const ShapeData &merge_data) if(conf_info.Obs->merge_flag == MergeType_Both || conf_info.Obs->merge_flag == MergeType_Engine) do_obs_merge_engine("", ""); - + // // Done // diff --git a/src/libcode/vx_shapedata/engine.h b/src/libcode/vx_shapedata/engine.h index 0b8241110a..50d67628cf 100644 --- a/src/libcode/vx_shapedata/engine.h +++ b/src/libcode/vx_shapedata/engine.h @@ -225,6 +225,7 @@ class ModeFuzzyEngine { // traditional version void do_obs_merging(const char *default_config, const char *merge_config); + // multivar version void do_obs_merging(const ShapeData &merge_data);