Skip to content

Commit

Permalink
Develop develop-ref after #2217 (#2219)
Browse files Browse the repository at this point in the history 
Co-authored-by: John Halley Gotway <johnhg@ucar.edu>
Co-authored-by: Julie Prestopnik <jpresto@seneca.rap.ucar.edu>
Co-authored-by: jprestop <jpresto@ucar.edu>
Co-authored-by: Howard Soh <hsoh@seneca.rap.ucar.edu>
Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com>
Co-authored-by: Randy Bullock <bullock@seneca.rap.ucar.edu>
Co-authored-by: davidfillmore <fillmore.winslow.david@gmail.com>
Co-authored-by: rgbullock <bullock@ucar.edu>
Co-authored-by: John Halley Gotway <johnhg@kiowa.rap.ucar.edu>
Co-authored-by: johnhg <johnhg@ucar.edu>
Co-authored-by: Seth Linden <linden@seneca.rap.ucar.edu>
Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Co-authored-by: Seth Linden <linden@kiowa.rap.ucar.edu>
Co-authored-by: Howard Soh <hsoh@kiowa.rap.ucar.edu>
Co-authored-by: Seth Linden <linden@ucar.edu>
Co-authored-by: hsoh-u <hsoh@ucar.edu>
Co-authored-by: John Halley Gotway <johnhg@seneca.rap.ucar.edu>
Co-authored-by: MET Tools Test Account <met_test@seneca.rap.ucar.edu>
Co-authored-by: mo-mglover <78152252+mo-mglover@users.noreply.github.com>
Co-authored-by: davidalbo <dave@ucar.edu>
Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com>
Co-authored-by: Dan Adriaansen <dadriaan@ucar.edu>
Co-authored-by: Dave Albo <dave@seneca.rap.ucar.edu>
Co-authored-by: Molly Smith <molly.b.smith@noaa.gov>
Co-authored-by: Jonathan Vigh <jvigh@ucar.edu>
  • Loading branch information
@JohnHalleyGotway @jprestop
@j-opatz @rgbullock @georgemccabe @sethlinden @hsoh-u @mo-mglover @davidalbo @lisagoodrich @DanielAdriaansen @mollybsmith-noaa @jvigh
24 people authored Aug 3, 2022
1 parent 811d5d8 commit db2d75a
Show file tree
Hide file tree
Showing 75 changed files with 1,065 additions and 654 deletions.
4 changes: 2 additions & 2 deletions data/config/IODA2NCConfig_default
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,8 +88,8 @@ obs_name_map = [];
// Default mapping for Metadata.
//
metadata_map = [
{ key = "message_type"; val = "msg_type"; },
{ key = "station_id"; val = "report_identifier"; },
{ key = "message_type"; val = "msg_type,station_ob"; },
{ key = "station_id"; val = "station_id,report_identifier"; },
{ key = "pressure"; val = "air_pressure,pressure"; },
{ key = "height"; val = "height,height_above_mean_sea_level"; },
{ key = "elevation"; val = ""; }
Expand Down
4 changes: 2 additions & 2 deletions docs/Users_Guide/reformat_point.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -951,8 +951,8 @@ _____________________
.. code-block:: none
metadata_map = [
{ key = "message_type"; val = "msg_type"; },
{ key = "station_id"; val = "report_identifier"; },
{ key = "message_type"; val = "msg_type,station_ob"; },
{ key = "station_id"; val = "station_id,report_identifier"; },
{ key = "pressure"; val = "air_pressure,pressure"; },
{ key = "height"; val = "height,height_above_mean_sea_level"; },
{ key = "elevation"; val = ""; }
Expand Down
174 changes: 140 additions & 34 deletions scripts/python/met_point_obs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
- This is the base class and the customized script should extend the met_point_obs.
- The customized script (for example "custom_reader") must implement "def read_data(self, args)"
which fills the array (list) variables at __init__().
which fills the array (python list or numpy array) variables at __init__().
- The args can be 1) single string argument, 2) the list of arguments, and 3) the dictionary of arguments.
- The variable "met_point_data" must be set for MET tools
- The customized script is expected to include following codes:
Expand All @@ -17,6 +17,7 @@
met_point_data = point_obs_data.get_point_data()
'''

import os
from abc import ABC, abstractmethod
import numpy as np

Expand All @@ -28,52 +29,128 @@ class met_point_obs(ABC):
'''
classdocs
'''
ERROR_P = " ==ERROR_PYTHON=="
INFO_P = " ==INFO_PYTHON=="

python_prefix = 'PYTHON_POINT_RAW'

def __init__(self, use_var_id=True):
'''
Constructor
'''
self.input_name = None
self.use_var_id = use_var_id # True if variable index, False if GRIB code

# Header
self.nhdr = 0
self.npbhdr = 0
self.nhdr_typ = 0 # type table
self.nhdr_sid = 0 #station_uid table
self.nhdr_sid = 0 # station_id table
self.nhdr_vld = 0 # valid time strings
self.hdr_typ = [] # (nhdr)
self.hdr_sid = [] # (nhdr)
self.hdr_vld = [] # (nhdr)
self.hdr_lat = [] # (nhdr)
self.hdr_lon = [] # (nhdr)
self.hdr_elv = [] # (nhdr)
self.hdr_typ_table = [] # (nhdr_typ, mxstr2)
self.hdr_sid_table = [] # (nhdr_sid, mxstr2)
self.hdr_vld_table = [] # (nhdr_vld, mxstr)
self.hdr_prpt_typ = [] # optional
self.hdr_irpt_typ = [] # optional
self.hdr_inst_typ = [] # optional
self.hdr_typ = [] # (nhdr) integer
self.hdr_sid = [] # (nhdr) integer
self.hdr_vld = [] # (nhdr) integer
self.hdr_lat = [] # (nhdr) float
self.hdr_lon = [] # (nhdr) float
self.hdr_elv = [] # (nhdr) float
self.hdr_typ_table = [] # (nhdr_typ, mxstr2) string
self.hdr_sid_table = [] # (nhdr_sid, mxstr2) string
self.hdr_vld_table = [] # (nhdr_vld, mxstr) string

#Observation data
self.nobs = 0
self.nobs_qty = 0
self.nobs_var = 0
self.obs_qty = [] # (nobs_qty )
self.obs_hid = [] # (nobs)
self.obs_vid = [] # (nobs) veriable index or GRIB code
self.obs_lvl = [] # (nobs)
self.obs_hgt = [] # (nobs)
self.obs_val = [] # (nobs)
self.obs_qty_table = [] # (nobs_qty, mxstr)
self.obs_var_table = [] # (nobs_var, mxstr2) optional if GRIB code at self.obs_vid
self.obs_qty = [] # (nobs_qty) integer, index of self.obs_qty_table
self.obs_hid = [] # (nobs) integer
self.obs_vid = [] # (nobs) integer, veriable index from self.obs_var_table or GRIB code
self.obs_lvl = [] # (nobs) float
self.obs_hgt = [] # (nobs) float
self.obs_val = [] # (nobs) float
self.obs_qty_table = [] # (nobs_qty, mxstr) string
self.obs_var_table = [] # (nobs_var, mxstr2) string, required if self.use_var_id is True

# Optional variables for PREPBUFR, not supported yet
self.hdr_prpt_typ = [] # optional
self.hdr_irpt_typ = [] # optional
self.hdr_inst_typ = [] # optional

@abstractmethod
def read_data(self, args):
# args should be list or dictionary
# args can be input_file_name, list, or dictionary
# - The variables at __init__ should be filled as python list or numpy array
# - set self.input_name
#
# Here is a template
'''
if isinstance(args, dict):
in_filename = args.get('in_name',None)
elif isinstance(args, list):
in_filename = args[0]
else:
in_filename = args
self.input_name = in_filename
'''
pass

def check_data_member_float(self, local_var, var_name):
if 0 == len(local_var):
self.log_error("{v} is empty".format(v=var_name))
elif isinstance(local_var, list):
if 0 <= str(type(local_var[0])).find('numpy'):
self.log_info("Recommend using numpy instead of python list for {v} ({t}) to avoid side effect".format(
v=var_name, t=type(local_var[0])))
elif not isinstance(local_var[0], (int, float)):
self.log_error("Not supported data type ({t}) for {v}[0] (int or float only)".format(
v=var_name, t=local_var[0].type))
elif not isinstance(local_var, np.ndarray):
self.log_error("Not supported data type ({t}) for {v} (list and numpy.ndarray)".format(
v=var_name, t=type(local_var)))

def check_data_member_int(self, local_var, var_name):
if 0 == len(local_var):
self.log_error("{v} is empty".format(v=var_name))
elif isinstance(local_var, list):
if 0 <= str(type(local_var[0])).find('numpy'):
self.log_info("Recommend using numpy instead of python list for {v} ({t}) to avoid side effect".format(
v=var_name, t=type(local_var[0])))
elif not isinstance(local_var[0], int):
self.log_error("Not supported data type ({t}) for {v}[0] (int only)".format(
v=var_name, t=type(local_var[0])))
elif not isinstance(local_var, np.ndarray):
self.log_error("Not supported data type ({t}) for {v} (list and numpy.ndarray)".format(
v=var_name, t=type(local_var)))

def check_data_member_string(self, local_var, var_name):
if 0 == len(local_var):
self.log_error("{v} is empty".format(v=var_name))
elif not isinstance(local_var, (list)):
self.log_error("Not supported data type ({t}) for {v} (list)".format(
v=var_name, t=type(local_var)))

def check_point_data(self):
if self.input_name is not None and not os.path.exists(self.input_name):
self.log_error('The netcdf input {f} does not exist'.format(f=self.input_name))
else:
self.check_data_member_int(self.hdr_typ,'hdr_typ')
self.check_data_member_int(self.hdr_sid,'hdr_sid')
self.check_data_member_int(self.hdr_vld,'hdr_vld')
self.check_data_member_float(self.hdr_lat,'hdr_lat')
self.check_data_member_float(self.hdr_lon,'hdr_lon')
self.check_data_member_float(self.hdr_elv,'hdr_elv')
self.check_data_member_string(self.hdr_typ_table,'hdr_typ_table')
self.check_data_member_string(self.hdr_sid_table,'hdr_sid_table')
self.check_data_member_string(self.hdr_vld_table,'hdr_vld_table')

self.check_data_member_int(self.obs_qty,'obs_qty')
self.check_data_member_int(self.obs_hid,'obs_hid')
self.check_data_member_int(self.obs_vid,'obs_vid')
self.check_data_member_float(self.obs_lvl,'obs_lvl')
self.check_data_member_float(self.obs_hgt,'obs_hgt')
self.check_data_member_float(self.obs_val,'obs_val')
self.check_data_member_string(self.obs_qty_table,'obs_qty_table')
self.check_data_member_string(self.obs_var_table,'obs_var_table')

def get_point_data(self):
if self.nhdr <= 0:
self.nhdr = len(self.hdr_lat)
Expand All @@ -91,8 +168,18 @@ def get_point_data(self):
self.nobs_qty = len(self.obs_qty_table)
if self.nobs_var <= 0:
self.nobs_var = len(self.obs_var_table)
self.check_point_data()
return self.__dict__

def log_error(self, err_msgs):
print(self.ERROR_P)
for err_line in err_msgs.split('\n'):
print('{p} {m}'.format(p=self.ERROR_P, m=err_line))
print(self.ERROR_P)

def log_info(self, info_msg):
print('{p} {m}'.format(p=self.INFO_P, m=info_msg))

@staticmethod
def is_python_script(arg_value):
return arg_value.startswith(met_point_obs.python_prefix)
Expand Down Expand Up @@ -120,31 +207,50 @@ def print_data(key, data_array, show_count=COUNT_SHOW):
def print_point_data(met_point_data, print_subset=True):
print(' === MET point data by python embedding ===')
if print_subset:
met_point_obs.print_data('nhdr', met_point_data['nhdr'])
met_point_obs.print_data('nobs' , met_point_data['nobs'])
met_point_obs.print_data('nhdr',met_point_data['nhdr'])
met_point_obs.print_data('nobs',met_point_data['nobs'])
met_point_obs.print_data('use_var_id',met_point_data['use_var_id'])
met_point_obs.print_data('hdr_typ',met_point_data['hdr_typ'])
met_point_obs.print_data('hdr_typ_table',met_point_data['hdr_typ_table'])
met_point_obs.print_data('hdr_sid',met_point_data['hdr_sid'])
met_point_obs.print_data('hdr_sid_table',met_point_data['hdr_sid_table'])
met_point_obs.print_data('hdr_vld',met_point_data['hdr_vld'])
met_point_obs.print_data('hdr_vld_table',met_point_data['hdr_vld_table'])
met_point_obs.print_data('hdr_lat',met_point_data['hdr_lat'])
met_point_obs.print_data('hdr_lon',met_point_data['hdr_lon'])
met_point_obs.print_data('hdr_elv',met_point_data['hdr_elv'])
met_point_obs.print_data('obs_hid',met_point_data['obs_hid'])
met_point_obs.print_data('obs_vid',met_point_data['obs_vid'])
met_point_obs.print_data('obs_val',met_point_data['obs_val'])
met_point_obs.print_data('obs_var_table',met_point_data['obs_var_table'])
met_point_obs.print_data('obs_qty',met_point_data['obs_qty'])
met_point_obs.print_data('obs_qty_table',met_point_data['obs_qty_table'])
met_point_obs.print_data('hdr_typ_table',met_point_data['hdr_typ_table'])
met_point_obs.print_data('hdr_sid_table',met_point_data['hdr_sid_table'])
met_point_obs.print_data('hdr_vld_table',met_point_data['hdr_vld_table'])
met_point_obs.print_data('obs_lvl',met_point_data['obs_lvl'])
met_point_obs.print_data('obs_hgt',met_point_data['obs_hgt'])
met_point_obs.print_data('obs_val',met_point_data['obs_val'])

else:
print('All',met_point_data)
print(" nhdr: ",met_point_data['nhdr'])
print(" nobs: ",met_point_data['nobs'])
print('use_var_id: ',met_point_data['use_var_id'])
print('hdr_typ: ',met_point_data['hdr_typ'])
print('obs_vid: ',met_point_data['obs_vid'])
print('obs_val: ',met_point_data['obs_val'])
print('obs_var_table: ',met_point_data['obs_var_table'])
print('obs_qty_table: ',met_point_data['obs_qty_table'])
print('hdr_typ_table: ',met_point_data['hdr_typ_table'])
print('hdr_sid: ',met_point_data['hdr_sid'])
print('hdr_sid_table: ',met_point_data['hdr_sid_table'])
print('hdr_vld: ',met_point_data['hdr_vld'])
print('hdr_vld_table: ',met_point_data['hdr_vld_table'])
print('hdr_lat: ',met_point_data['hdr_lat'])
print('hdr_lon: ',met_point_data['hdr_lon'])
print('hdr_elv: ',met_point_data['hdr_elv'])
print('obs_hid: ',met_point_data['obs_hid'])
print('obs_vid: ',met_point_data['obs_vid'])
print('obs_var_table: ',met_point_data['obs_var_table'])
print('obs_qty: ',met_point_data['obs_qty'])
print('obs_qty_table: ',met_point_data['obs_qty_table'])
print('obs_lvl: ',met_point_data['obs_lvl'])
print('obs_hgt: ',met_point_data['obs_hgt'])
print('obs_val: ',met_point_data['obs_val'])

print(' === MET point data by python embedding ===')


Expand All @@ -160,9 +266,9 @@ def read_data(self, arg_map={}):
self.hdr_lon = np.array([ -89., -89., -89., -89., -89., -89., -89., -89., -89., -92., -92., -92., -92., -89., -89., -89., -89., -89., -89., -89., -89., -89., -92., -92., -92., -92. ])
self.hdr_elv = np.array([ 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220., 220. ])

self.obs_qid = np.array([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])
self.obs_hid = np.array([ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 25 ])
self.obs_vid = np.array([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ])
self.obs_qty = np.array([ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])
self.obs_lvl = np.array([ 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000., 1000. ])
self.obs_hgt = np.array([ 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2., 2. ])
self.obs_val = np.array([ 292., 292.5, 293., 293.5, 294., 294.5, 295., 295.5, 296., 292., 293.4, 293., 296., 294., 92., 92.5, 93., 93.5, 94., 94.5, 95., 95.5, 96., 92., 93.4, 93., 96., 94. ])
Expand Down
17 changes: 11 additions & 6 deletions src/basic/vx_config/dictionary.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1949,7 +1949,8 @@ if ( Entry ) {

mlog << Warning
<< "\nDictionary::lookup_thresh() -> "
<< "threshold lookup failed for name \"" << name << "\"\n\n";
<< "threshold lookup failed for name \""
<< name << "\"\n\n";

}
}
Expand All @@ -1962,7 +1963,9 @@ if ( !Entry || !is_correct_type ) {

mlog << Error
<< "\nDictionary::lookup_thresh() -> "
<< "lookup failed for name \"" << name << "\"\n\n";
<< "threshold \"" << name
<< "\" must start with a valid comparison operator "
<< "(<, <=, ==, !=, >, >=, lt, le, eq, ne, gt, ge).\n\n";

exit ( 1 );

Expand Down Expand Up @@ -2008,7 +2011,6 @@ if ( Entry ) {
}
}


LastLookupStatus = is_correct_type;

if ( !Entry || !is_correct_type ) {
Expand All @@ -2017,7 +2019,9 @@ if ( !Entry || !is_correct_type ) {

mlog << Error
<< "\nDictionary::lookup_thresh_array() -> "
<< "lookup failed for name \"" << name << "\"\n\n";
<< "each threshold in array \"" << name
<< "\" must start with a valid comparison operator "
<< "(<, <=, ==, !=, >, >=, lt, le, eq, ne, gt, ge).\n\n";

exit ( 1 );

Expand Down Expand Up @@ -2056,8 +2060,9 @@ if ( Dict->n_entries() > 0 ) {

mlog << Error
<< "\nDictionary::lookup_thresh_array() -> "
<< "array \"" << name
<< "\" does not contain ThreshType.\n\n";
<< "each threshold in array \"" << name
<< "\" must start with a valid comparison operator "
<< "(<, <=, ==, !=, >, >=, lt, le, eq, ne, gt, ge).\n\n";

exit ( 1 );

Expand Down
1 change: 1 addition & 0 deletions src/basic/vx_log/Makefile.am
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ libvx_log_a_SOURCES = concat_string.cc concat_string.h \
file_fxns.cc file_fxns.h \
indent.cc indent.h \
logger.cc logger.h \
main.cc main.h \
string_array.cc string_array.h \
str_wrappers.cc str_wrappers.h \
vx_log.h
Expand Down
12 changes: 8 additions & 4 deletions src/basic/vx_log/Makefile.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ am__v_AR_1 =
libvx_log_a_AR = $(AR) $(ARFLAGS)
libvx_log_a_LIBADD =
am_libvx_log_a_OBJECTS = concat_string.$(OBJEXT) file_fxns.$(OBJEXT) \
indent.$(OBJEXT) logger.$(OBJEXT) string_array.$(OBJEXT) \
str_wrappers.$(OBJEXT)
indent.$(OBJEXT) logger.$(OBJEXT) main.$(OBJEXT) \
string_array.$(OBJEXT) str_wrappers.$(OBJEXT)
libvx_log_a_OBJECTS = $(am_libvx_log_a_OBJECTS)
AM_V_P = $(am__v_P_@AM_V@)
am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
Expand All @@ -128,8 +128,8 @@ depcomp = $(SHELL) $(top_srcdir)/depcomp
am__maybe_remake_depfiles = depfiles
am__depfiles_remade = ./$(DEPDIR)/concat_string.Po \
./$(DEPDIR)/file_fxns.Po ./$(DEPDIR)/indent.Po \
./$(DEPDIR)/logger.Po ./$(DEPDIR)/str_wrappers.Po \
./$(DEPDIR)/string_array.Po
./$(DEPDIR)/logger.Po ./$(DEPDIR)/main.Po \
./$(DEPDIR)/str_wrappers.Po ./$(DEPDIR)/string_array.Po
am__mv = mv -f
CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \
$(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS)
Expand Down Expand Up @@ -336,6 +336,7 @@ libvx_log_a_SOURCES = concat_string.cc concat_string.h \
file_fxns.cc file_fxns.h \
indent.cc indent.h \
logger.cc logger.h \
main.cc main.h \
string_array.cc string_array.h \
str_wrappers.cc str_wrappers.h \
vx_log.h
Expand Down Expand Up @@ -392,6 +393,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/file_fxns.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/indent.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/logger.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/main.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/str_wrappers.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/string_array.Po@am__quote@ # am--include-marker

Expand Down Expand Up @@ -547,6 +549,7 @@ distclean: distclean-am
-rm -f ./$(DEPDIR)/file_fxns.Po
-rm -f ./$(DEPDIR)/indent.Po
-rm -f ./$(DEPDIR)/logger.Po
-rm -f ./$(DEPDIR)/main.Po
-rm -f ./$(DEPDIR)/str_wrappers.Po
-rm -f ./$(DEPDIR)/string_array.Po
-rm -f Makefile
Expand Down Expand Up @@ -598,6 +601,7 @@ maintainer-clean: maintainer-clean-am
-rm -f ./$(DEPDIR)/file_fxns.Po
-rm -f ./$(DEPDIR)/indent.Po
-rm -f ./$(DEPDIR)/logger.Po
-rm -f ./$(DEPDIR)/main.Po
-rm -f ./$(DEPDIR)/str_wrappers.Po
-rm -f ./$(DEPDIR)/string_array.Po
-rm -f Makefile
Expand Down
Loading

0 comments on commit db2d75a

Please sign in to comment.