Feature 2772 python embedding fill value (#2810)

* #2772 Update masked value to -9999. Renamed error_messageg to error_message

* #2772 Renamed error_messageg to error_message

* #2772 Accept 4th argument as user defined fill value.

* #2772 Added python_numpy_plot_data_plane_missing which processes user defined fill value

---------

Co-authored-by: Howard Soh <hsoh@seneca.rap.ucar.edu>

internal/test_unit/xml/unit_python.xml

@@ -115,6 +115,25 @@
     </output>
   </test>
 
+  <!-- Invokes Python script that reads in NUMPY text data (convert data value 0.0 to missing value) -->
+  <test name="python_numpy_plot_data_plane_missing">
+    <exec>&MET_BIN;/plot_data_plane</exec>
+    <env>
+      <pair><name>MET_PYTHON_EXE</name> <value>&MET_PYTHON_EXE;</value></pair>
+    </env>
+    <param> \
+      PYTHON_NUMPY \
+      &OUTPUT_DIR;/python/letter_numpy_0_to_missing.ps \
+      'name = "&MET_BASE;/python/examples/read_ascii_numpy.py &DATA_DIR_PYTHON;/letter.txt LETTER 0.0";' \
+      -plot_range 0.0 255.0 \
+      -title "Python enabled numpy plot_data_plane" \
+      -v 1
+    </param>
+    <output>
+      <ps>&OUTPUT_DIR;/python/letter_numpy_0_to_missing.ps</ps>
+    </output>
+  </test>
+
   <test name="python_numpy_plot_data_plane_file_type">
     <exec>&MET_BIN;/plot_data_plane</exec>
     <param> \

scripts/python/examples/read_ascii_numpy.py

@@ -1,5 +1,6 @@
 import os
 import sys
+import numpy
 from met.dataplane import dataplane
 
 ###########################################
@@ -67,33 +68,47 @@ def set_dataplane_attrs():
 ##  load the data into the numpy array
 ##
 
-if len(sys.argv) != 3:
-   dataplane.quit("read_ascii_numpy.py -> Must specify exactly one input file and a name for the data.")
+SCRIPT_NAME = "read_ascii_numpy.py ->"
+if len(sys.argv) < 3:
+   dataplane.quit(f"{SCRIPT_NAME} Must specify exactly one input file and a name for the data.")
+elif len(sys.argv) > 4:
+   dataplane.quit(f"{SCRIPT_NAME} Have not supported arguments [{sys.argv[4:]}]")
 
 # Read the input file as the first argument
 input_file = os.path.expandvars(sys.argv[1])
 data_name  = sys.argv[2]
 
 try:
-   log("Input File:\t" + repr(input_file))
-   log("Data Name:\t" + repr(data_name))
+   user_fill_value = None
+   try:
+      if len(sys.argv) > 3:
+         user_fill_value = float(sys.argv[3])
+   except:
+      log(f"{SCRIPT_NAME} Ignored argument {sys.argv[3]}")
+      pass
+
+   log(f"{SCRIPT_NAME} Input File:\t{repr(input_file)}")
+   log(f"{SCRIPT_NAME} Data Name:\t{repr(data_name)}")
    if os.path.exists(input_file):
       # read_2d_text_input() reads n by m text data and returns 2D numpy array
       met_data = dataplane.read_2d_text_input(input_file)
       if met_data is None:
-         dataplane.quit(f" Fail to build met_data from {input_file}")
+         dataplane.quit(f"{SCRIPT_NAME} Fail to build met_data from {input_file}")
       else:
-         log("Data Shape:\t" + repr(met_data.shape))
-         log("Data Type:\t" + repr(met_data.dtype))
+         log(f"{SCRIPT_NAME} Data Shape:\t{repr(met_data.shape)}")
+         log(f"{SCRIPT_NAME} Data Type:\t{repr(met_data.dtype)}")
+         if user_fill_value is not None:
+            met_data = numpy.ma.masked_values(met_data, user_fill_value)
+         log(f"{SCRIPT_NAME} Python Type:\t{type(met_data)}")
    else:
-      dataplane.quit(f"input {input_file} does exist!!!")
+      dataplane.quit(f"{SCRIPT_NAME} input {input_file} does exist!!!")
 except:
    import traceback
    traceback.print_exc()
-   dataplane.quit(f"Unknown error with {sys.argv[0]}: ")
+   dataplane.quit(f"{SCRIPT_NAME} Unknown error with {sys.argv[0]}: ")
 
 attrs = set_dataplane_attrs()
-log("Attributes:\t" + repr(attrs))
+log(f"{SCRIPT_NAME} Attributes:\t{repr(attrs)}")
 
 # Sets fill_value if it exists at the dataplane
 #attrs['fill_value'] = 255  # for letter.txt

scripts/python/met/logger.py

@@ -18,7 +18,7 @@ def append_error_prompt(msg):
       return f'{logger.ERROR_P}: {msg}'
 
    @staticmethod
-   def error_messageg(msg):
+   def error_message(msg):
       msgs = msg if isinstance(msg, list) else [msg]
       msgs.insert(0, '')
       msgs.append('')
@@ -62,7 +62,7 @@ def get_met_fill_value(self):
       return met_base.MET_FILL_VALUE
 
    def error_msg(self, msg):
-      logger.error_messageg(msg)
+      logger.error_message(msg)
 
    def get_prompt(self):
       return met_base_tools.get_prompt()
@@ -116,7 +116,7 @@ def convert_to_array(ndarray_data):
             for byte_data in ndarray_data:
                array_data.append(byte_data.decode("utf-8").rstrip())
       elif isinstance(ndarray_data, (np.ma.MaskedArray, np.ma.core.MaskedArray)):
-         array_data = np.ma.getdata(ndarray_data, subok=False).tolist()
+         array_data = ndarray_data.filled(fill_value=-9999).tolist()
       elif isinstance(ndarray_data, np.ndarray):
          array_data = ndarray_data.tolist()
       else:
@@ -126,7 +126,7 @@ def convert_to_array(ndarray_data):
    @staticmethod
    def convert_to_ndarray(array_data):
       if isinstance(array_data, (np.ma.MaskedArray, np.ma.core.MaskedArray)):
-         ndarray_data = np.ma.getdata(array_data, subok=False)
+         ndarray_data = array_data.filled(fill_value=-9999)
       elif isinstance(array_data, np.ndarray):
          ndarray_data = array_data
       else:

scripts/python/met/point.py

@@ -797,7 +797,7 @@ def convert_point_data(point_data, check_all_records=False, input_type='csv'):
       csv_point_data.check_csv_point_data(check_all_records)
       tmp_point_data = csv_point_data.get_point_data()
    else:
-      met_base.error_messageg(f'convert_point_data(() Not supported input type: {input_type}')
+      met_base.error_message(f'convert_point_data(() Not supported input type: {input_type}')
    return tmp_point_data
 
 def get_empty_point_obs():