dtcenter · JohnHalleyGotway · Feb 2, 2024 · Nov 20, 2023 · Nov 20, 2023 · Nov 27, 2023
diff --git a/docs/Users_Guide/config_options.rst b/docs/Users_Guide/config_options.rst
@@ -494,6 +494,49 @@ Where code is running in a production context, it is worth being familiar with
 the binding / affinitization method on the particular system and building it
 into any relevant scripting.
 
+.. _met_keep_temp_file:
+
+MET_KEEP_TEMP_FILE
+------------------
+
+The MET_KEEP_TEMP_FILE environment variable can be set to control the runtime
+behavior of the MET tools. The MET tools write temporary files in several places
+in the application and library code. By default, those temporary files are deleted
+when they are no longer needed. However it can be useful for development, testing,
+and debugging to keep them for further inspection. Setting this environment variable
+to a value of :code:`yes` or :code:`true` instructs the MET tools to retain temporary
+files instead of deleting them.
+
+Note that doing so may fill up the temporary directory. It is the responsiblity of
+the user to monitor the temporary directory usage and remove temporary files that
+are no longer needed.
+
+When running with this option, users are advised to refer to section
+:numref:`config_tmp_dir` and write temporary files to a personal location rather than
+the default shared :code:`/tmp` directory.
+
+.. _met_python_debug:
+
+MET_PYTHON_DEBUG
+----------------
+
+The MET_PYTHON_DEBUG environment variable can be set to enable debugging log messages
+related to Python embedding. These log messages are disabled by default. The environment
+variable can be set to a value of :code:`all` for all log messages, :code:`dataplane`
+for log messages when reading gridded data, or :code:`point` for log messages when
+reading point data.
+
+.. _met_python_tmp_format:
+
+MET_PYTHON_TMP_FORMAT
+---------------------
+
+The MET_PYTHON_TMP_FORMAT environment variable defines whether temporary files for
+Python embedding should be written as NetCDF files or using JSON/NumPy serialization.
+By default, they are written using JSON for attributes and NumPy serialization for data
+to avoid NetCDF library conflicts between MET and Python. Setting this environment
+variable to :code:`netcdf` enables the use of temporary NetCDF files instead.
+
 Settings Common to Multiple Tools
 =================================
 
@@ -3770,13 +3813,22 @@ Where "job_name" is set to one of the following:
 
 * "filter"
 
-  To filter out the STAT or TCMPR lines matching the job filtering
-  criteria specified below and using the optional arguments below.
+  To filter out the STAT lines matching the job filtering criteria
+  specified below and using the optional arguments below.
   The output STAT lines are written to the file specified using the
   "-dump_row" argument.
+
   Required Args: -dump_row
 
-|    
+  Optional Args:
+
+  .. code-block:: none
+
+    -set_hdr column_name value
+       May be used multiple times to override data written to the
+       output dump_row file.
+
+|
 
 * "summary"
 
@@ -3805,8 +3857,8 @@ Where "job_name" is set to one of the following:
 
   * Format the -column option as LINE_TYPE:COLUMN.
 
-|     
-    
+|
+
   Use the -derive job command option to automatically derive
   statistics on the fly from input contingency tables and partial
   sums.
@@ -3832,10 +3884,14 @@ Where "job_name" is set to one of the following:
 
   .. code-block:: none
 
-    -by column_name to specify case information
-    -out_alpha to override default alpha value of 0.05
-    -derive to derive statistics on the fly
-    -column_union to summarize multiple columns
+    -by column_name
+       To specify case information.
+    -out_alpha
+       To override the default alpha value.
+    -derive
+       To derive statistics on the fly.
+    -column_union
+       To summarize multiple columns.
 
 * "aggregate"
 
@@ -3852,8 +3908,8 @@ Where "job_name" is set to one of the following:
                ISC, ECNT, RPS, RHIST, PHIST, RELP, SSVAR
 
   Required Args: -line_type
-  
-| 
+
+|
 
 * "aggregate_stat"
 
@@ -3887,8 +3943,8 @@ Where "job_name" is set to one of the following:
   .. code-block:: none
 
     -out_thresh or -out_fcst_thresh and -out_obs_thresh
-     When -out_line_type FHO, CTC, CTS, MCTC, MCTS,
-                         PCT, PSTD, PJC, PRC
+       When -out_line_type FHO, CTC, CTS, MCTC, MCTS,
+                           PCT, PSTD, PJC, PRC
 
   Additional Optional Args for -line_type MPR:
 
@@ -3901,14 +3957,14 @@ Where "job_name" is set to one of the following:
     -out_obs_wind_thresh
     -out_wind_logic
     When -out_line_type WDIR
-	    
+
   Additional Optional Arg for:
 
   .. code-block:: none
 
     -line_type ORANK -out_line_type PHIST, SSVAR ...
     -out_bin_size
-	    
+
   Additional Optional Args for:
 
   .. code-block:: none
@@ -3917,14 +3973,14 @@ Where "job_name" is set to one of the following:
     -out_eclv_points
 
 * "ss_index"
-  
+
   The skill score index job can be configured to compute a weighted
   average of skill scores derived from a configurable set of
   variables, levels, lead times, and statistics. The skill score
   index is computed using two models, a forecast model and a
   reference model. For each statistic in the index, a skill score
   is computed as:
-  
+
   SS = 1 - (S[model]*S[model])/(S[reference]*S[reference])
 
   Where S is the statistic.
@@ -4135,17 +4191,19 @@ Where "job_name" is set to one of the following:
     "-rank_corr_flag  value"
     "-vif_flag        value"
 
-  For aggregate and aggregate_stat job types:
-
   .. code-block:: none
 
-    "-out_stat        path"   to write a .stat output file for the job
-                              including the .stat header columns. Multiple
-                              values for each header column are written as
-                              a comma-separated list.
-    "-set_hdr col_name value" may be used multiple times to explicity
-                              specify what should be written to the header
-                              columns of the output .stat file.
+    -out_stat path
+       To write a .stat output file for aggregate and aggregate_stat jobs
+       including the .stat header columns. Multiple input values for each
+       header column are written to the output as a comma-separated list
+       of unique values.
+
+    -set_hdr col_name value
+       May be used multiple times to explicity specify what should be
+       written to the header columns of the output .stat file for
+       aggregate and aggregate_stat jobs or output dump_row file
+       for filter jobs.
 
   When using the "-by" job command option, you may reference those columns
   in the "-set_hdr" job command options. For example, when computing statistics

diff --git a/docs/Users_Guide/stat-analysis.rst b/docs/Users_Guide/stat-analysis.rst
@@ -604,7 +604,7 @@ The Stat-Analysis tool supports several additional job command options which may
 This job command option is extremely useful. It can be used multiple times to specify a list of STAT header column names. When reading each input line, the Stat-Analysis tool concatenates together the entries in the specified columns and keeps track of the unique cases. It applies the logic defined for that job to each unique subset of data. For example, if your output was run over many different model names and masking regions, specify **-by MODEL,VX_MASK** to get output for each unique combination rather than having to run many very similar jobs.
 
 .. code-block:: none
-		
+
   -column_min     col_name value
   -column_max     col_name value
   -column_eq      col_name value
@@ -615,30 +615,30 @@ This job command option is extremely useful. It can be used multiple times to sp
 The column filtering options may be used when the **-line_type** has been set to a single value. These options take two arguments, the name of the data column to be used followed by a value, string, or threshold to be applied. If multiple column_min/max/eq/thresh/str options are listed, the job will be performed on their intersection. Each input line is only retained if its value meets the numeric filtering criteria defined, matches one of the strings defined by the **-column_str** option, or does not match any of the string defined by the **-column_str_exc** option. Multiple filtering strings may be listed using commas. Defining thresholds in MET is described in :numref:`config_options`.
 
 .. code-block:: none
-		
+
   -dump_row file
 
 Each analysis job is performed over a subset of the input data. Filtering the input data down to a desired subset is often an iterative process. The **-dump_row** option may be used for each job to specify the name of an output file to which the exact subset of data used for that job will be written. When initially constructing Stat-Analysis jobs, users are strongly encouraged to use the option and check its contents to ensure that the analysis was actually done over the intended subset.
 
 .. code-block:: none
-		
+
   -out_line_type name
 
 This option specifies the desired output line type(s) for the **aggregate_stat** job type.
 
 .. code-block:: none
-		
+
   -out_stat file
   -set_hdr  col_name string
 
 The Stat-Analysis tool writes its output to either the log file or the file specified using the **-out** command line option. However the **aggregate** and **aggregate_stat** jobs create STAT output lines and the standard output written lacks the full set of STAT header columns. The **-out_stat** job command option may be used for these jobs to specify the name of an output file to which full STAT output lines should be written. When the **-out_stat** job command option is used for **aggregate** and **aggregate_stat** jobs the output is sent to the **-out_stat** file instead of the log or **-out** file.
 
-Jobs will often combine output with multiple entries in the header columns. For example, a job may aggregate output with three different values in the **VX_MASK** column, such as "mask1", "mask2", and "mask3". The output **VX_MASK** column will contain the unique values encountered concatenated together with commas: "mask1,mask2,mask3". Alternatively, the **-set_hdr** option may be used to specify what should be written to the output header columns, such as "-set_hdr VX_MASK all_three_masks".
+Jobs will often combine output with multiple entries in the header columns. For example, a job may aggregate output with three different values in the **VX_MASK** column, such as "mask1", "mask2", and "mask3". The output **VX_MASK** column will contain the unique values encountered concatenated together with commas: "mask1,mask2,mask3". Alternatively, the **-set_hdr** option may be used to specify what should be written to the output header columns, such as "-set_hdr VX_MASK all_three_masks". When **-set_hdr** is specified for **filter** jobs, it controls what is written to the **-dump_row** output file.
 
 When using the "-out_stat" option to create a .stat output file and stratifying results using one or more "-by" job command options, those columns may be referenced in the "-set_hdr" option. When using multiple "-by" options, use "CASE" to reference the full case information string:
 
 .. code-block:: none
-		
+
   -job aggregate_stat -line_type MPR -out_line_type CNT -by FCST_VAR,OBS_SID \
   -set_hdr VX_MASK OBS_SID -set_hdr DESC CASE
 
@@ -662,7 +662,7 @@ When processing input MPR lines, these options may be used to define a masking g
 When processing input MPR lines, these options are used to define the forecast, observation, or both thresholds to be applied when computing statistics. For categorical output line types (FHO, CTC, CTS, MCTC, MCTS) these define the categorical thresholds. For continuous output line types (SL1L2, SAL1L2, CNT), these define the continuous filtering thresholds and **-out_cnt_logic** defines how the forecast and observed logic should be combined.
 
 .. code-block:: none
-		
+
   -out_fcst_wind_thresh thresh
   -out_obs_wind_thresh  thresh
   -out_wind_thresh      thresh

diff --git a/internal/test_unit/R_test/test_util.R b/internal/test_unit/R_test/test_util.R
@@ -383,7 +383,10 @@ compareStatLty = function(stat1, stat2, lty, verb=0, strict=0){
 	# compare the information in the header columns
 	for(intCol in 2:21){
 		listMatch = apply(data.frame(dfV1[,intCol], dfV2[,intCol]), 1,
-				function(a){ a[1] == a[2] });
+				function(a){
+					same = (a[1] == a[2]) | (is.na(a[1]) & is.na(a[2]));
+					same[is.na(same)] = FALSE;
+					return(same); });
 		intNumDiff = sum( !listMatch[ !is.na(listMatch) ] );
 		if( 0 < intNumDiff ){
 			if( 1 <= verb ){

diff --git a/internal/test_unit/xml/unit_python.xml b/internal/test_unit/xml/unit_python.xml
@@ -206,6 +206,7 @@
   <test name="python_numpy_point_stat">
     <exec>&MET_BIN;/point_stat</exec>
     <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
       <pair><name>FCST_COMMAND</name> <value>&MET_BASE;/python/examples/read_ascii_numpy.py &DATA_DIR_PYTHON;/fcst.txt FCST</value></pair>
       <pair><name>OBS_COMMAND</name>  <value>&MET_BASE;/python/examples/read_ascii_numpy.py &DATA_DIR_PYTHON;/obs.txt OBS</value></pair>
     </env>
@@ -480,21 +481,6 @@
     </output>
   </test>
 
-  <test name="python_point2grid_pb2nc_TMP">
-    <exec>&MET_BIN;/point2grid</exec>
-    <param> \
-      'PYTHON_NUMPY=&MET_BASE;/python/examples/read_met_point_obs.py &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc' \
-      G212 \
-      &OUTPUT_DIR;/python/pb2nc_TMP.nc \
-      -field 'name="TMP"; level="*"; valid_time="20120409_120000"; censor_thresh=[ &lt;0 ]; censor_val=[0];' \
-      -name TEMP \
-      -v 1
-    </param>
-    <output>
-      <grid_nc>&OUTPUT_DIR;/python/pb2nc_TMP.nc</grid_nc>
-    </output>
-  </test>
-
   <!-- Invokes user-python logic to read a point obs -->
   <test name="python_point2grid_pb2nc_TMP_user_python">
     <exec>&MET_BIN;/point2grid</exec>
@@ -535,6 +521,7 @@
   <test name="python_plot_point_obs_CONFIG">
     <exec>&MET_BIN;/plot_point_obs</exec>
     <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
       <pair><name>TO_GRID</name> <value>NONE</value></pair>
     </env>
     <param> \
@@ -561,6 +548,7 @@
                 > &OUTPUT_DIR;/python/ensemble_stat/input_file_list; \
           &MET_BIN;/ensemble_stat</exec>
     <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
       <pair><name>DESC</name>           <value>NA</value></pair>
       <pair><name>OBS_ERROR_FLAG</name> <value>FALSE</value></pair>
       <pair><name>SKIP_CONST</name>     <value>FALSE</value></pair>
@@ -587,6 +575,7 @@
   <test name="python_point_stat_GRIB1_NAM_GDAS_WINDS">
     <exec>&MET_BIN;/point_stat</exec>
     <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
       <pair><name>BEG_DS</name>         <value>-1800</value></pair>
       <pair><name>END_DS</name>         <value>1800</value></pair>
       <pair><name>OUTPUT_PREFIX</name>  <value>GRIB1_NAM_GDAS_WINDS</value></pair>
@@ -605,6 +594,9 @@
   </test>
 
   <test name="python_plot_data_plane_SEMILATLON_ZONAL_MEAN">
+    <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
+    </env>
     <exec>&MET_BIN;/plot_data_plane</exec>
     <param> \
     PYTHON_NUMPY \
@@ -619,6 +611,9 @@
   </test>
 
   <test name="python_pcp_combine_SEMILATLON_MERIDIONAL_MEAN">
+    <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
+    </env>
     <exec>&MET_BIN;/pcp_combine</exec>
     <param> \
     -add PYTHON_NUMPY \

diff --git a/internal/test_unit/xml/unit_stat_analysis_ps.xml b/internal/test_unit/xml/unit_stat_analysis_ps.xml
@@ -76,6 +76,7 @@
       -job filter -line_type MPR -fcst_var TMP -fcst_lev Z2 -vx_mask DTC165 \
       -column_str     OBS_SID KDLN,KDHT,KDEN,KDLS,KDMA,KDMN,KDVT,KDEW \
       -column_str_exc OBS_SID KDLN,KDHT \
+      -set_hdr DESC FILTER_OBS_SID \
       -dump_row &OUTPUT_DIR;/stat_analysis_ps/POINT_STAT_FILTER_OBS_SID.stat \
       -v 1
     </param>

diff --git a/scripts/python/examples/read_ascii_numpy.py b/scripts/python/examples/read_ascii_numpy.py
@@ -5,7 +5,7 @@
 ###########################################
 
 def log(msg):
-   dataplane.log_msg(msg)
+   dataplane.log_message(msg)
 
 def set_dataplane_attrs():
    # attrs is a dictionary which contains attributes describing the dataplane.
@@ -95,5 +95,5 @@ def set_dataplane_attrs():
 attrs = set_dataplane_attrs()
 log("Attributes:\t" + repr(attrs))
 
-# Sets fill_value if it exists
+# Sets fill_value if it exists at the dataplane
 #attrs['fill_value'] = 255  # for letter.txt
diff --git a/scripts/python/examples/read_ascii_numpy_grid.py b/scripts/python/examples/read_ascii_numpy_grid.py
@@ -27,8 +27,11 @@
    met_data = dataplane.read_2d_text_input(input_file)
    print("Data Shape:\t" + repr(met_data.shape))
    print("Data Type:\t" + repr(met_data.dtype))
-except NameError:
-   print("Can't find the input file")
+except NameError as ex:
+   print(" === ERROR from read_ascii_numpy_grid.py")
+   print(f"   Exception: {type(ex)}  {ex}")
+   print(f"   sys.argv: {sys.argv}")
+   print("   Can't find the input file")
 
 # attrs is a dictionary which contains attributes describing the dataplane.
 # attrs should have 9 items, each of data type string:

diff --git a/scripts/python/examples/read_ascii_xarray.py b/scripts/python/examples/read_ascii_xarray.py
@@ -6,7 +6,7 @@
 ###########################################
 
 def log(msg):
-   dataplane.log_msg(msg)
+   dataplane.log_message(msg)
 
 log("Python Script:\t" + repr(sys.argv[0]))