feature 1695 fix issues with ensemble changes

met/src/tools/core/ensemble_stat/ensemble_stat.cc

@@ -62,7 +62,8 @@
 //   031    09/13/21  Seth Linden    Changed obs_qty to obs_qty_inc.
 //                    Added code for obs_qty_exc.
 //   032    10/07/21  Halley Gotway  MET #1905 Add -ctrl option.
-//   032    11/15/21  Halley Gotway  MET #1968 Ensemble -ctrl error check.
+//   033    11/15/21  Halley Gotway  MET #1968 Ensemble -ctrl error check.
+//   034    01/14/21  McCabe         MET #1695 All members in one file.
 //
 ////////////////////////////////////////////////////////////////////////
 
@@ -302,7 +303,7 @@ void process_command_line(int argc, char **argv) {
    // Copy ensemble file list to forecast file list
    fcst_file_list = ens_file_list;
 
-   // Prepend the control member, if specified
+   // Append the control member, if specified
    if(ctrl_file.nonempty()) {
 
       if(ens_file_list.has(ctrl_file) && n_ens_files != 1) {
@@ -312,10 +313,9 @@ void process_command_line(int argc, char **argv) {
          exit(1);
       }
 
-      ctrl_index = 0;
-
-      // Add control file to beginning of forecast file list
-      fcst_file_list.insert(ctrl_index, ctrl_file.c_str());
+      // Add control member file to end of the forecast file list
+      fcst_file_list.add(ctrl_file.c_str());
+      ctrl_file_index = fcst_file_list.n()-1;
    }
 
    // Check that the end_ut >= beg_ut
@@ -948,8 +948,12 @@ void process_vx() {
       // Process masks Grids and Polylines in the config file
       conf_info.process_masks(grid);
 
+      // Determine the index of the control member in list of data values
+      int ctrl_data_index = (is_bad_data(ctrl_file_index) ?
+                             bad_data_int : fcst_file_vld.sum()-1);
+
       // Setup the PairDataEnsemble objects
-      conf_info.set_vx_pd(n_vx_vld, ctrl_index);
+      conf_info.set_vx_pd(n_vx_vld, ctrl_data_index);
 
       // Process the point observations
       if(point_obs_flag) process_point_vx();
@@ -1200,7 +1204,7 @@ int process_point_ens(int i_ens, int &n_miss) {
 
    mlog << Debug(2) << "\n" << sep_str << "\n\n"
         << "Processing " << file_type << " file: " << ens_file
-        << (i_ens == ctrl_index ? " (control)\n" : "\n");
+        << (i_ens == ctrl_file_index ? " (control)\n" : "\n");
 
    // Loop through each of the fields to be verified and extract
    // the forecast fields for verification
@@ -2012,7 +2016,7 @@ void process_grid_scores(int i_vx,
 
             // Store the unperturbed ensemble value
             // Exclude the control member from the variance
-            if(j != ctrl_index) pd.add_ens_var_sums(i, fraw_dp[j](x, y));
+            if(j != ctrl_file_index) pd.add_ens_var_sums(i, fraw_dp[j](x, y));
          }
       } // end for j
 

met/src/tools/core/ensemble_stat/ensemble_stat.h

@@ -107,8 +107,8 @@ static bool         ens_mean_flag; // Flag for ensemble mean processing
 static ConcatString ens_mean_user; // User-specified ensemble mean data file
 static ConcatString ens_mean_file; // Computed ensemble mean output file
 
-static ConcatString ctrl_file;     // Control member
-static int          ctrl_index = bad_data_int; // Control member index
+static ConcatString ctrl_file;     // Control member file
+static int          ctrl_file_index = bad_data_int; // Control member file index
 
 // Input Observation files
 static StringArray  grid_obs_file_list;

met/src/tools/core/ensemble_stat/ensemble_stat_conf_info.cc

@@ -212,6 +212,14 @@ void EnsembleStatConfInfo::process_config(GrdFileType etype,
          exit(1);
       }
 
+      // The control ID must be set when the control file is specified
+      if(control_id.empty() && use_ctrl) {
+         mlog << Error << "\nEnsembleStatConfInfo::process_config() -> "
+              << "the control_id must be set if processing a single input "
+              << "file with the -ctrl option\n\n";
+         exit(1);
+      }
+
       // If control ID is set, it cannot be found in ens_member_ids
       if(!control_id.empty() && ens_member_ids.has(control_id)) {
          mlog << Error << "\nEnsembleStatConfInfo::process_config() -> "
@@ -323,6 +331,9 @@ void EnsembleStatConfInfo::process_config(GrdFileType etype,
       ens_input.push_back(ens_info);
    } // end for i
 
+   // Unset MET_ENS_MEMBER_ID that was previously set
+   unsetenv(met_ens_member_id);
+
    // Conf: ens.ens_thresh
    vld_ens_thresh = conf.lookup_double(conf_key_ens_ens_thresh);
 
@@ -757,7 +768,6 @@ void EnsembleStatVxOpt::process_config(GrdFileType ftype, Dictionary &fdict,
                                        StringArray * fcst_files,
                                        bool use_ctrl, ConcatString control_id) {
    int i, j;
-   int file_index_start = 0;
    VarInfoFactory info_factory;
    map<STATLineType,STATOutputType>output_map;
    Dictionary *dict;
@@ -770,11 +780,11 @@ void EnsembleStatVxOpt::process_config(GrdFileType ftype, Dictionary &fdict,
    // Allocate new EnsVarInfo object for fcst
    vx_pd.fcst_info = new EnsVarInfo();
 
-   // Add control member as first input
-   if(use_ctrl) {
+   // Loop over ensemble member IDs to substitute
+   for(i=0; i<ens_member_ids.n(); i++) {
 
-      // Set environment variable for ens member ID
-      setenv(met_ens_member_id, control_id.c_str(), 1);
+      // set environment variable for ens member ID
+      setenv(met_ens_member_id, ens_member_ids[i].c_str(), 1);
 
       // Allocate new VarInfo object
       next_var = info_factory.new_var_info(ftype);
@@ -787,15 +797,22 @@ void EnsembleStatVxOpt::process_config(GrdFileType ftype, Dictionary &fdict,
       input_info.file_list = fcst_files;
       vx_pd.fcst_info->add_input(input_info);
 
-      // Change the starting index to the first non-control file
-      file_index_start = 1;
-   }
+      // Add InputInfo to fcst info list for each ensemble file provided
+      // set var_info to NULL to note first VarInfo should be used
+      int last_member_index = fcst_files->n() - (use_ctrl ? 1 : 0);
+      for(j=1; j<last_member_index; j++) {
+         input_info.var_info = NULL;
+         input_info.file_index = j;
+         input_info.file_list = fcst_files;
+         vx_pd.fcst_info->add_input(input_info);
+      } // end for j
+   } // end for i
 
-   // Loop over ensemble member IDs to substitute
-   for(i=0; i<ens_member_ids.n(); i++) {
+   // Add control member as the last input
+   if(use_ctrl) {
 
-      // set environment variable for ens member ID
-      setenv(met_ens_member_id, ens_member_ids[i].c_str(), 1);
+      // Set environment variable for ens member ID
+      setenv(met_ens_member_id, control_id.c_str(), 1);
 
       // Allocate new VarInfo object
       next_var = info_factory.new_var_info(ftype);
@@ -804,20 +821,10 @@ void EnsembleStatVxOpt::process_config(GrdFileType ftype, Dictionary &fdict,
       next_var->set_dict(fdict);
 
       input_info.var_info = next_var;
-      input_info.file_index = file_index_start;
+      input_info.file_index = fcst_files->n() - 1;
       input_info.file_list = fcst_files;
       vx_pd.fcst_info->add_input(input_info);
-
-      // Add InputInfo to fcst info list for each ensemble file provided
-      // set var_info to NULL to note first VarInfo should be used
-      for(j=file_index_start+1; j<fcst_files->n(); j++) {
-         input_info.var_info = NULL;
-         input_info.file_index = j;
-         input_info.file_list = fcst_files;
-         vx_pd.fcst_info->add_input(input_info);
-      } // end for j
-
-   } // end for i
+   }
 
    // Allocate new VarInfo object for obs
    vx_pd.obs_info  = info_factory.new_var_info(otype);

met/src/tools/other/gen_ens_prod/gen_ens_prod.cc

@@ -16,6 +16,7 @@
 //   ----   ----      ----           -----------
 //   000    09/10/21  Halley Gotway  MET #1904 Initial version.
 //   001    11/15/21  Halley Gotway  MET #1968 Ensemble -ctrl error check.
+//   002    01/14/21  McCabe         MET #1695 All members in one file.
 //
 ////////////////////////////////////////////////////////////////////////
 
@@ -186,7 +187,7 @@ void process_command_line(int argc, char **argv) {
    etype = ens_mtddf->file_type();
 
    // Process the configuration
-   conf_info.process_config(etype, &ens_files);
+   conf_info.process_config(etype, &ens_files, ctrl_file.nonempty());
 
    // Allocate arrays to store threshold counts
    thresh_cnt_na       = new NumArray   [conf_info.get_max_n_cat()];

met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.cc

@@ -100,7 +100,7 @@ void GenEnsProdConfInfo::read_config(const ConcatString default_file_name,
 
 ////////////////////////////////////////////////////////////////////////
 
-void GenEnsProdConfInfo::process_config(GrdFileType etype, StringArray * ens_files) {
+void GenEnsProdConfInfo::process_config(GrdFileType etype, StringArray * ens_files, bool use_ctrl) {
    int i, j;
    VarInfoFactory info_factory;
    Dictionary *edict = (Dictionary *) 0;
@@ -146,6 +146,14 @@ void GenEnsProdConfInfo::process_config(GrdFileType etype, StringArray * ens_fil
          exit(1);
       }
 
+      // The control ID must be set when the control file is specified
+      if(control_id.empty() && use_ctrl) {
+         mlog << Error << "\nGenEnsProdConfInfo::process_config() -> "
+              << "the control_id must be set if processing a single input "
+              << "file with the -ctrl option\n\n";
+         exit(1);
+      }
+
       // If control ID is set, it cannot be found in ens_member_ids
       if(!control_id.empty() && ens_member_ids.has(control_id)) {
           mlog << Error << "\nGenEnsProdConfInfo::process_config() -> "

met/src/tools/other/gen_ens_prod/gen_ens_prod_conf_info.h

@@ -99,7 +99,7 @@ class GenEnsProdConfInfo {
       void clear();
 
       void read_config   (const ConcatString, const ConcatString);
-      void process_config(GrdFileType, StringArray *);
+      void process_config(GrdFileType, StringArray *, bool);
 
       GenEnsProdNcOutInfo parse_nc_info(Dictionary *);
 

scripts/environment/development.seneca

@@ -27,7 +27,7 @@ export MET_PYTHON_CC="-I${MET_PYTHON}/include/python3.8"
 export MET_PYTHON_LD="-L${MET_PYTHON}/lib -lpython3.8 -lcrypt -lpthread -ldl -lutil -lm"
 
 # -D__64BIT__ is required because we've compiled libgrib2c.a with that flag
-export CFLAGS="-DUNDERSCORE -fPIC -D__64BIT__"
+export CFLAGS="-DUNDERSCORE -fPIC -D__64BIT__ -g"
 export CXXFLAGS=${CFLAGS}
 
 # Set LDFLAGS to include -rpath settings when compiling MET

test/config/EnsembleStatConfig_single_file_grib

@@ -285,13 +285,13 @@ ensemble_flag = {
    latlon    = TRUE;
    mean      = TRUE;
    stdev     = TRUE;
-   minus     = TRUE;
-   plus      = TRUE;
+   minus     = FALSE;
+   plus      = FALSE;
    min       = TRUE;
    max       = TRUE;
-   range     = TRUE;
+   range     = FALSE;
    vld_count = TRUE;
-   frequency = TRUE;
+   frequency = FALSE;
    nep       = FALSE;
    nmep      = FALSE;
    rank      = TRUE;

test/config/EnsembleStatConfig_single_file_nc

@@ -9,7 +9,7 @@
 //
 // Output model name to be written
 //
-model = "WRF";
+model = "CFS";
 
 //
 // Output description to be written
@@ -46,19 +46,26 @@ censor_val    = [];
 cat_thresh    = [];
 nc_var_str    = "";
 
+//
+// Override the NetCDF CFS variable names and times 
+//
+file_type      = NETCDF_NCCF;
+set_attr_init  = "19820101";
+set_attr_valid = "19820701";
+set_attr_name  = "TMP";
+set_attr_level = "Z2";
+
 //
 // Ensemble product fields to be processed
 //
 ens = {
-   file_type = NETCDF_NCCF;
    ens_thresh = 1.0;
    vld_thresh = 1.0;
 
    field = [
       {
-         name       = "fcst";
-         level      = "(MET_ENS_MEMBER_ID,0,*,*)";
-         cat_thresh = [ >0.0, >=5.0 ];
+         name  = "fcst";
+         level = "(MET_ENS_MEMBER_ID,6,*,*)";
       }
    ];
 }
@@ -103,22 +110,19 @@ nmep_smooth = {
 // Forecast and observation fields to be verified
 //
 fcst = {
-   file_type = NETCDF_NCCF;
    field = [
       {
-         name       = "fcst";
-         level      = "(MET_ENS_MEMBER_ID,0,*,*)";
-         cat_thresh = [ >0.0, >=5.0 ];
+         name  = "fcst";
+         level = "(MET_ENS_MEMBER_ID,6,*,*)";
       }
    ];
 }
 
 obs = {
    field = [
       {
-         name       = "fcst";
-         level      = "(0,0,*,*)";
-         cat_thresh = [ >0.0, >=5.0 ];
+         name  = "fcst";
+         level = "(0,6,*,*)";
       }
    ];
 }
@@ -283,13 +287,13 @@ ensemble_flag = {
    latlon    = TRUE;
    mean      = TRUE;
    stdev     = TRUE;
-   minus     = TRUE;
-   plus      = TRUE;
+   minus     = FALSE;
+   plus      = FALSE;
    min       = TRUE;
    max       = TRUE;
-   range     = TRUE;
+   range     = FALSE;
    vld_count = TRUE;
-   frequency = TRUE;
+   frequency = FALSE;
    nep       = FALSE;
    nmep      = FALSE;
    rank      = TRUE;

test/xml/unit_ensemble_stat.xml

@@ -266,7 +266,9 @@
       -outdir &OUTPUT_DIR;/ensemble_stat -v 1
     </param>
     <output>
-      <grid_nc>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_NC_NO_CTRL_19700101_000000V_ens.nc</grid_nc>
+      <stat>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_NC_NO_CTRL_19820701_000000V.stat</stat>
+      <grid_nc>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_NC_NO_CTRL_19820701_000000V_ens.nc</grid_nc>
+      <grid_nc>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_NC_NO_CTRL_19820701_000000V_orank.nc</grid_nc>
     </output>
   </test>
 
@@ -287,7 +289,9 @@
       -outdir &OUTPUT_DIR;/ensemble_stat -v 1
     </param>
     <output>
-      <grid_nc>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_NC_WITH_CTRL_19700101_000000V_ens.nc</grid_nc>
+      <stat>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_NC_WITH_CTRL_19820701_000000V.stat</stat>
+      <grid_nc>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_NC_WITH_CTRL_19820701_000000V_ens.nc</grid_nc>
+      <grid_nc>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_NC_WITH_CTRL_19820701_000000V_orank.nc</grid_nc>
     </output>
   </test>
 
@@ -307,7 +311,9 @@
       -outdir &OUTPUT_DIR;/ensemble_stat -v 1
     </param>
     <output>
+      <stat>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_GRIB_NO_CTRL_20160608_060000V.stat</stat>
       <grid_nc>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_GRIB_NO_CTRL_20160608_060000V_ens.nc</grid_nc>
+      <grid_nc>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_GRIB_NO_CTRL_20160608_060000V_orank.nc</grid_nc>
     </output>
   </test>
 
@@ -328,7 +334,9 @@
       -outdir &OUTPUT_DIR;/ensemble_stat -v 1
     </param>
     <output>
+      <stat>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_GRIB_WITH_CTRL_20160608_060000V.stat</stat>
       <grid_nc>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_GRIB_WITH_CTRL_20160608_060000V_ens.nc</grid_nc>
+      <grid_nc>&OUTPUT_DIR;/ensemble_stat/ensemble_stat_SINGLE_FILE_GRIB_WITH_CTRL_20160608_060000V_orank.nc</grid_nc>
     </output>
   </test>