diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e18cbf4cca..8de167ba60 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,7 +7,7 @@ version: 2 # Build all formats (htmlzip, pdf, epub) #formats: all -formats: [pdf] +#formats: [pdf] # Optionally set the version of Python and requirements required to build your # docs diff --git a/data/wrappers/Makefile.am b/data/wrappers/Makefile.am index d2f83ff125..deb919438e 100644 --- a/data/wrappers/Makefile.am +++ b/data/wrappers/Makefile.am @@ -24,8 +24,10 @@ wrappers_DATA = \ set_python_env.py \ read_tmp_dataplane.py \ read_tmp_ascii.py \ + read_tmp_point_nc.py \ write_tmp_dataplane.py \ write_tmp_point.py \ + write_tmp_point_nc.py \ write_tmp_mpr.py EXTRA_DIST = ${wrappers_DATA} diff --git a/data/wrappers/Makefile.in b/data/wrappers/Makefile.in index 9ab052cca5..35b600e0c9 100644 --- a/data/wrappers/Makefile.in +++ b/data/wrappers/Makefile.in @@ -359,8 +359,10 @@ wrappers_DATA = \ set_python_env.py \ read_tmp_dataplane.py \ read_tmp_ascii.py \ + read_tmp_point_nc.py \ write_tmp_dataplane.py \ write_tmp_point.py \ + write_tmp_point_nc.py \ write_tmp_mpr.py EXTRA_DIST = ${wrappers_DATA} diff --git a/data/wrappers/read_tmp_point_nc.py b/data/wrappers/read_tmp_point_nc.py new file mode 100644 index 0000000000..8afaf16684 --- /dev/null +++ b/data/wrappers/read_tmp_point_nc.py @@ -0,0 +1,30 @@ +######################################################################## +# +# Reads temporary file into memory. +# +# usage: /path/to/python read_tmp_dataplane.py dataplane.tmp +# +######################################################################## + +import os +import sys + +met_base_dir = os.getenv('MET_BASE',None) +if met_base_dir is not None: + sys.path.append(os.path.join(met_base_dir, 'python')) + +# add share/met/python directory to system path to find met_point_obs +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, 'python'))) +from met_point_obs import met_point_obs +from met_point_obs_nc import nc_point_obs + +netcdf_filename = sys.argv[1] + +# read NetCDF file +print('{p} reading{f}'.format(p=met_point_obs.get_prompt(), f=netcdf_filename)) +point_obs_data = nc_point_obs() +point_obs_data.read_data(netcdf_filename) + +met_point_data = point_obs_data.get_point_data() +met_point_data['met_point_data'] = point_obs_data diff --git a/data/wrappers/write_tmp_point_nc.py b/data/wrappers/write_tmp_point_nc.py new file mode 100644 index 0000000000..1dbda19807 --- /dev/null +++ b/data/wrappers/write_tmp_point_nc.py @@ -0,0 +1,59 @@ +######################################################################## +# +# Adapted from a script provided by George McCabe +# Adapted by Randy Bullock +# +# usage: /path/to/python write_tmp_point.py \ +# tmp_output_filename .py +# +######################################################################## + +import os +import sys +import importlib.util + +met_base_dir = os.getenv('MET_BASE',None) +if met_base_dir is not None: + sys.path.append(os.path.join(met_base_dir, 'python')) + +# add share/met/python directory to system path to find met_point_obs +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), + os.pardir, 'python'))) + +from met_point_obs import met_point_obs +from met_point_obs_nc import nc_point_obs + +PROMPT = met_point_obs.get_prompt() +print("{p} Python Script:\t".format(p=PROMPT) + repr(sys.argv[0])) +print("{p} User Command:\t".format(p=PROMPT) + repr(' '.join(sys.argv[2:]))) +print("{p} Temporary File:\t".format(p=PROMPT) + repr(sys.argv[1])) + +tmp_filename = sys.argv[1] +pyembed_module_name = sys.argv[2] +sys.argv = sys.argv[2:] + +# append user script dir to system path +pyembed_dir, pyembed_file = os.path.split(pyembed_module_name) +if pyembed_dir: + sys.path.insert(0, pyembed_dir) + +if not pyembed_module_name.endswith('.py'): + pyembed_module_name += '.py' + +user_base = os.path.basename(pyembed_module_name).replace('.py','') + +spec = importlib.util.spec_from_file_location(user_base, pyembed_module_name) +met_in = importlib.util.module_from_spec(spec) +spec.loader.exec_module(met_in) + +if hasattr(met_in, 'point_obs_data'): + met_in.point_obs_data.save_ncfile(tmp_filename) +else: + if hasattr(met_in.met_point_data, 'point_obs_data'): + met_in.met_point_data['point_obs_data'].save_ncfile(tmp_filename) + else: + tmp_point_obs = nc_point_obs() + tmp_point_obs.put_data(met_in.met_point_data) + tmp_point_obs.save_ncfile(tmp_filename) + +#print('{p} writing {f}'.format(p=PROMPT, f=tmp_filename)) diff --git a/docs/Users_Guide/release-notes.rst b/docs/Users_Guide/release-notes.rst index 4dfd494c4a..a57773b4b3 100644 --- a/docs/Users_Guide/release-notes.rst +++ b/docs/Users_Guide/release-notes.rst @@ -9,6 +9,26 @@ When applicable, release notes are followed by the GitHub issue number which des enhancement, or new feature (`MET GitHub issues `_). Important issues are listed **in bold** for emphasis. +MET Version 11.0.1-rc1 release notes (20230221) +----------------------------------------------- + +* Bugfixes: + + * Bugfix: Fix the MET CF-Compliant NetCDF library code to Polar Stereographic data from NSIDC Sea Ice Edge NetCDF files (`#2218 `_). + * Bugfix: Remove override keyword to avoid C++11 dependency (`#2380 `_). + * Bugfix: Fix ASCII2NC to not compute AOD 550 if other inputs are negative values (`#2383 `_). + * Bugfix: Fix PB2NC to report accurate total observation counts in log messages (`#2387 `_). + * Bugfix: Update the MET flowchart for version 11.0.0 (`#2389 `_). + * Bugfix: Fix issues with the met_compile_all.sh script and associated tar files (`#2390 `_). + * Bugfix: Correct definitions of NCEP grid numbers 172 and 220 (`#2399 `_). + * Bugfix: Address MET-11.0.0 SonarQube Blocker Bugs (`#2402 `_). + * Bugfix: Refine fix for handling empty configuration files (`#2408 `_). + * Bugfix: Fix time interpolation of monthly climatology data between December 15 and January 15 (`#2412 `_). + * Bugfix: Fix ASCII2NC to handle missing NDBC buoy location information (`#2426 `_). + * Bugfix: Fix the MET vx_pointdata_python library to handle MET_PYTHON_EXE for python embedding of point observations (`#2428 `_). + * Bugfix: Refine the regrid dictionary's data conversion and censoring operations and fix climo time matching logic for a single monthly climo file (`#2437 `_). + * Bugfix: Fix inconsistent ASCII2NC AIRNOW location lookup logic (`#2452 `_). + MET Version 11.0.0 release notes (20221209) ------------------------------------------- diff --git a/docs/conf.py b/docs/conf.py index 055e2c9651..56667e56f9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,11 +20,11 @@ project = 'MET' author = 'UCAR/NCAR, NOAA, CSU/CIRA, and CU/CIRES' author_list = 'Opatz, J., T. Jensen, J. Prestopnik, H. Soh, L. Goodrich, B. Brown, R. Bullock, J. Halley Gotway, K. Newman' -version = '11.0.0' +version = '11.0.1-rc1' verinfo = version release = f'{version}' -release_year = '2022' -release_date = f'{release_year}-12-09' +release_year = '2023' +release_date = f'{release_year}-02-21' copyright = f'{release_year}, {author}' # -- General configuration --------------------------------------------------- diff --git a/internal/scripts/installation/compile_MET_all.sh b/internal/scripts/installation/compile_MET_all.sh index 0a7907d63a..4ebc7c704e 100644 --- a/internal/scripts/installation/compile_MET_all.sh +++ b/internal/scripts/installation/compile_MET_all.sh @@ -832,9 +832,9 @@ if [ $COMPILE_MET -eq 1 ]; then # https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html # ${parameter:+word} # If parameter is null or unset, nothing is substituted, otherwise the expansion of word is substituted. - export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_DIR}/lib${MET_NETCDF:+:$MET_NETCDF/lib}${MET_HDF5:+:$MET_HDF5/lib}${MET_BUFRLIB:+:$MET_BUFRLIB}${MET_GRIB2CLIB:+:$MET_GRIB2CLIB}${MET_PYTHON:+:$MET_PYTHON/lib}${MET_GSL:+:$MET_GSL/lib}" + export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_DIR}/lib${ADDTL_DIR:+:$ADDTL_DIR}${MET_NETCDF:+:$MET_NETCDF/lib}${MET_HDF5:+:$MET_HDF5/lib}${MET_BUFRLIB:+:$MET_BUFRLIB}${MET_GRIB2CLIB:+:$MET_GRIB2CLIB}${MET_PYTHON:+:$MET_PYTHON/lib}${MET_GSL:+:$MET_GSL/lib}" export LDFLAGS="${LDFLAGS} -Wl,-rpath,${LIB_JASPER:+$LIB_JASPER}${LIB_LIBPNG:+:$LIB_PNG}${LIB_Z:+$LIB_Z}" - export LDFLAGS="${LDFLAGS} ${LIB_JASPER:+-L$LIB_JASPER} ${LIB_LIBPNG:+-L$LIB_LIBPNG} ${MET_HDF5:+-L$MET_HDF5/lib}" + export LDFLAGS="${LDFLAGS} ${LIB_JASPER:+-L$LIB_JASPER} ${LIB_LIBPNG:+-L$LIB_LIBPNG} ${MET_HDF5:+-L$MET_HDF5/lib} ${ADDTL_DIR:+-L$ADDTL_DIR}" export LIBS="${LIBS} -lhdf5_hl -lhdf5 -lz" export MET_FONT_DIR=${TEST_BASE}/fonts diff --git a/internal/scripts/installation/config/install_met_env.acorn_py3.10 b/internal/scripts/installation/config/install_met_env.acorn_py3.10 new file mode 100644 index 0000000000..b97699a187 --- /dev/null +++ b/internal/scripts/installation/config/install_met_env.acorn_py3.10 @@ -0,0 +1,36 @@ +module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304/ +module load intel python/3.10.4 +module load ve/evs/1.0 +module load netcdf/4.7.4 +module load hdf5/1.12.2 +module load bufr/11.5.0 +module load zlib/1.2.11 +module load jasper/2.0.25 +module load libpng/1.6.37 +module load gsl/2.7 +module load g2c/1.6.4 + +export TEST_BASE=/apps/sw_review/emc/MET/11.0.1 +export LIB_DIR=${TEST_BASE}/external_libs +export BIN_DIR_PATH=${TEST_BASE}/exec +export COMPILER=intel_19.1.3.304 +export MET_SUBDIR=${TEST_BASE} +export MET_TARBALL=v11.0.1.tar.gz +export USE_MODULES=TRUE +export ADDTL_DIR=/apps/spack/gettext/0.21/intel/19.1.3.304/at2kdo4edvuhyzrt5g6zhwrdb7bdui4s/lib64 +export PYTHON_MODULE=python_3.10.4 +export MET_PYTHON=/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp +export MET_PYTHON_CC=-I/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/include/python3.10 +export MET_PYTHON_LD=-L/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/lib/python3.10/config-3.10-x86_64-linux-gnu/\ -L/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/lib64\ -lpython3.10\ -lintl\ -lcrypt\ -ldl\ -lutil\ -lm\ -lm +export MET_NETCDF=/apps/prod/hpc-stack/intel-19.1.3.304/netcdf/4.7.4 +export MET_HDF5=/apps/prod/hpc-stack/intel-19.1.3.304/hdf5/1.12.2 +export MET_BUFRLIB=/apps/ops/prod/libs/intel/19.1.3.304/bufr/11.5.0/lib64 +export MET_GRIB2CLIB=/apps/ops/prod/libs/intel/19.1.3.304/g2c/1.6.4/lib64 +export MET_GRIB2CINC=/apps/ops/prod/libs/intel/19.1.3.304/g2c/1.6.4/include +export MET_GSL=/apps/spack/gsl/2.7/intel/19.1.3.304/xks7dxbowrdxhjck5zxc4rompopocevb +export BUFRLIB_NAME=-lbufr_4 +export GRIB2CLIB_NAME=-lg2c +export LIB_JASPER=/apps/spack/jasper/2.0.25/intel/19.1.3.304/sjib74krrorkyczqpqah4tvewmlnqdx4/lib64 +export LIB_LIBPNG=/apps/spack/libpng/1.6.37/intel/19.1.3.304/4ohkronuhlyherusoszzrmur5ewvlwzh/lib +export LIB_Z=/apps/spack/zlib/1.2.11/intel/19.1.3.304/hjotqkckeoyt6j6tibalwzrlfljcjtdh/lib +export SET_D64BIT=FALSE \ No newline at end of file diff --git a/internal/scripts/installation/config/install_met_env.wcoss2_py3.10 b/internal/scripts/installation/config/install_met_env.wcoss2_py3.10 new file mode 100644 index 0000000000..c6d550c3a3 --- /dev/null +++ b/internal/scripts/installation/config/install_met_env.wcoss2_py3.10 @@ -0,0 +1,47 @@ +# JY module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304/ +module load intel +# JY add following two +module load craype/2.7.13 +module load cray-mpich/8.1.12 + +module load python/3.10.4 +module load netcdf/4.7.4 +module load hdf5/1.10.6 +module load bufr/11.6.0 +module load zlib/1.2.11 +module load jasper/2.0.25 +module load libpng/1.6.37 +module load gsl/2.7 +module load g2c/1.6.4 + +#export TEST_BASE=/apps/ops/para/libs/intel/19.1.3.304/met/11.0.1 +export TEST_BASE=$(pwd) +export LIB_DIR=${TEST_BASE}/external_libs +export BIN_DIR_PATH=${TEST_BASE}/bin +export COMPILER=intel_19.1.3.304 +export MET_SUBDIR=${TEST_BASE} +export MET_TARBALL=v11.0.1.tar.gz +export USE_MODULES=TRUE +export ADDTL_DIR=/apps/spack/gettext/0.21/intel/19.1.3.304/at2kdo4edvuhyzrt5g6zhwrdb7bdui4s/lib64 +export PYTHON_MODULE=python_3.10.4 +export MET_PYTHON=/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp +export MET_PYTHON_CC=-I/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/include/python3.10 +export MET_PYTHON_LD=-L/apps/spack/python/3.10.4/intel/19.1.3.304/xqft4d45h4dp4xnbz2ue3nbxv65i6bgp/lib\ -lpython3.10\ -lpthread\ -ldl\ -lutil\ -lm\ -Xlinker\ -export-dynamic +export MET_NETCDF=/apps/prod/hpc-stack/intel-19.1.3.304/netcdf/4.7.4 +# JY export MET_HDF5=/apps/prod/hpc-stack/intel-19.1.3.304/hdf5/1.12.2 +export MET_HDF5=${HDF5_ROOT} +export MET_BUFRLIB=/apps/ops/prod/libs/intel/19.1.3.304/bufr/11.6.0/lib64 +# JY export MET_GRIB2CLIB=/apps/ops/prod/libs/intel/19.1.3.304/g2c/1.6.4/lib64 +# JY export MET_GRIB2CINC=/apps/ops/prod/libs/intel/19.1.3.304/g2c/1.6.4/include +export MET_GRIB2CLIB=${g2c_ROOT}/lib64 +export MET_GRIB2CINC=${G2C_INC} +export MET_GSL=/apps/spack/gsl/2.7/intel/19.1.3.304/xks7dxbowrdxhjck5zxc4rompopocevb +export BUFRLIB_NAME=-lbufr_4 +export GRIB2CLIB_NAME=-lg2c +# JY export LIB_JASPER=/apps/spack/jasper/2.0.25/intel/19.1.3.304/sjib74krrorkyczqpqah4tvewmlnqdx4/lib64 +export LIB_JASPER=${JASPER_LIBDIR} +# JY export LIB_LIBPNG=/apps/spack/libpng/1.6.37/intel/19.1.3.304/4ohkronuhlyherusoszzrmur5ewvlwzh/lib +export LIB_LIBPNG=${LIBPNG_LIBDIR} +# JY export LIB_Z=/apps/spack/zlib/1.2.11/intel/19.1.3.304/hjotqkckeoyt6j6tibalwzrlfljcjtdh/lib +export LIB_Z=${ZLIB_LIBDIR} +export SET_D64BIT=FALSE diff --git a/internal/scripts/installation/modulefiles/11.0.1.lua.wcoss2 b/internal/scripts/installation/modulefiles/11.0.1.lua.wcoss2 new file mode 100644 index 0000000000..c4afdd9fda --- /dev/null +++ b/internal/scripts/installation/modulefiles/11.0.1.lua.wcoss2 @@ -0,0 +1,27 @@ +help([[ +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +local hierA = hierarchyA(pkgNameVer,1) +local compNameVer = hierA[1] + + +conflict(pkgName) + +local opt = os.getenv("HPC_OPT") or os.getenv("OPT") or "/opt/modules" + +local base = pathJoin(opt,compNameVer,pkgName,pkgVersion) + +prepend_path("PATH", pathJoin(base,"bin")) + +setenv("MET_ROOT", base) +setenv("MET_BASE", pathJoin(base, "share", "met")) +setenv("MET_VERSION", pkgVersion) + +whatis("Name: ".. pkgName) +whatis("Version: " .. pkgVersion) +whatis("Category: applications") +whatis("Description: Model Evaluation Tools (MET)") diff --git a/internal/scripts/installation/modulefiles/11.0.1_acorn b/internal/scripts/installation/modulefiles/11.0.1_acorn new file mode 100644 index 0000000000..832194dc76 --- /dev/null +++ b/internal/scripts/installation/modulefiles/11.0.1_acorn @@ -0,0 +1,34 @@ +#%Module###################################################################### +## +## Model Evaluation Tools +## +proc ModulesHelp { } { + puts stderr "Sets up the paths and environment variables to use the Model Evaluation Tools v11.0.1 + *** For help see the official MET webpage at http://www.dtcenter.org/met/users ***" +} + +# The intel compiler is required to run MET + +module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304/ +module load intel python/3.10.4 +module load ve/evs/1.0 +module load netcdf/4.7.4 +module load hdf5/1.12.2 +module load bufr/11.5.0 +module load zlib/1.2.11 +module load jasper/2.0.25 +module load libpng/1.6.37 +module load gsl/2.7 +module load g2c/1.6.4 + +set base /apps/sw_review/emc/MET/11.0.1 +set ver 11.0.1 +set share $base/share/met +set lib_base $base + +prepend-path PATH $base/exec + +setenv METversion V$ver +setenv MET_ROOT $base + + diff --git a/internal/test_unit/xml/unit_python.xml b/internal/test_unit/xml/unit_python.xml index 05f9d2650d..5a519d9212 100644 --- a/internal/test_unit/xml/unit_python.xml +++ b/internal/test_unit/xml/unit_python.xml @@ -495,6 +495,25 @@ + + + &MET_BIN;/point2grid + + MET_PYTHON_EXE &MET_PYTHON_EXE; + + \ + 'PYTHON_NUMPY=&MET_BASE;/python/read_met_point_obs.py &OUTPUT_DIR;/pb2nc/ndas.20120409.t12z.prepbufr.tm00.nc' \ + G212 \ + &OUTPUT_DIR;/python/pb2nc_TMP_user_python.nc \ + -field 'name="TMP"; level="*"; valid_time="20120409_120000"; censor_thresh=[ <0 ]; censor_val=[0];' \ + -name TEMP \ + -v 1 + + + &OUTPUT_DIR;/python/pb2nc_TMP_user_python.nc + + + &MET_BIN;/plot_point_obs diff --git a/scripts/python/Makefile.am b/scripts/python/Makefile.am index 76aa04cdf1..689708e4c3 100644 --- a/scripts/python/Makefile.am +++ b/scripts/python/Makefile.am @@ -27,6 +27,7 @@ pythonscriptsdir = $(pkgdatadir)/python pythonscripts_DATA = \ met_point_obs.py \ + met_point_obs_nc.py \ read_ascii_numpy.py \ read_ascii_numpy_grid.py \ read_ascii_xarray.py \ diff --git a/scripts/python/Makefile.in b/scripts/python/Makefile.in index 80da2e8f84..64470101b4 100644 --- a/scripts/python/Makefile.in +++ b/scripts/python/Makefile.in @@ -298,6 +298,7 @@ top_srcdir = @top_srcdir@ pythonscriptsdir = $(pkgdatadir)/python pythonscripts_DATA = \ met_point_obs.py \ + met_point_obs_nc.py \ read_ascii_numpy.py \ read_ascii_numpy_grid.py \ read_ascii_xarray.py \ diff --git a/scripts/python/met_point_obs.py b/scripts/python/met_point_obs.py index 5cfc5f7141..458a91572c 100755 --- a/scripts/python/met_point_obs.py +++ b/scripts/python/met_point_obs.py @@ -1,21 +1,25 @@ - +#!/usr/bin/env python3 ''' Created on Nov 10, 2021 @author: hsoh - This is the base class and the customized script should extend the met_point_obs. -- The customized script (for example "custom_reader") must implement "def read_data(self, args)" - which fills the array (python list or numpy array) variables at __init__(). -- The args can be 1) single string argument, 2) the list of arguments, and 3) the dictionary of arguments. -- The variable "met_point_data" must be set for MET tools +- The customized script (for example "custom_reader") must implement + "def read_data(self, args)" which fills the array variables at __init__(). +- The args can be 1) single string argument, 2) the list of arguments, + or 3) the dictionary of arguments. +- A python objects, met_point_data, must set: + + "point_obs_data" is an optional to use custom python EXE. + It's a python instance which processes the point observation data - The customized script is expected to include following codes: # prepare arguments for the customized script args = {'input', sys.argv[1]} # or args = [] point_obs_data = custom_reader() - point_obs_data.read_data() + point_obs_data.read_data(args) met_point_data = point_obs_data.get_point_data() + ''' import os @@ -24,22 +28,29 @@ COUNT_SHOW = 30 -MET_PYTHON_OBS_ARGS = "MET_POINT_PYTHON_ARGS" +def get_prompt(): + return " python:" + +def met_is_python_prefix(user_cmd): + return user_cmd.startswith(base_met_point_obs.python_prefix) class base_met_point_obs(object): ''' classdocs ''' - ERROR_P = " ==ERROR_PYTHON==" - INFO_P = " ==INFO_PYTHON==" + ERROR_P = " ==PYTHON_ERROR==" + INFO_P = " ==PYTHON_INFO==" + + python_prefix = 'PYTHON_POINT_USER' - python_prefix = 'PYTHON_POINT_RAW' + FILL_VALUE = -9999. def __init__(self, use_var_id=True): ''' Constructor ''' + self.count_info = "" self.input_name = None self.ignore_input_file = False self.use_var_id = use_var_id # True if variable index, False if GRIB code @@ -74,6 +85,8 @@ def __init__(self, use_var_id=True): self.obs_val = [] # (nobs) float self.obs_qty_table = [] # (nobs_qty, mxstr) string self.obs_var_table = [] # (nobs_var, mxstr2) string, required if self.use_var_id is True + self.obs_var_unit = [] # (nobs_var, mxstr2) string, optional if self.use_var_id is True + self.obs_var_desc = [] # (nobs_var, mxstr3) string, optional if self.use_var_id is True # Optional variables for PREPBUFR, not supported yet self.hdr_prpt_typ = [] # optional @@ -98,17 +111,12 @@ def check_data_member_float(self, local_var, var_name): self.add_error_msg("{v} is empty (float)".format(v=var_name)) elif isinstance(local_var, list): if isinstance(local_var[0], str) and not self.is_number(local_var[0]): - self.add_error_msg("Not supported data type ({v} string) for {n}[0] (int or float only)".format( + self.add_error_msg("Not supported data type: {n}[0]={v}, string type, not a number (int or float only)".format( n=var_name, v=local_var[0])) - elif 0 <= str(type(local_var[0])).find('numpy'): - self.log_info("Recommend using numpy instead of python list for {v} ({t}) to avoid side effect".format( - v=var_name, t=type(local_var[0]))) - elif not isinstance(local_var[0], (int, float)): - # self.add_error_msg("Not supported data type ({t}) for {v}[0] (int or float only)".format( - # v=var_name, t=local_var[0].type)) + elif 0 > str(type(local_var[0])).find('numpy') and not isinstance(local_var[0], (int, float)): self.add_error_msg("Not supported data type ({t}) for {v}[0] (int or float only)".format( v=var_name, t=type(local_var[0]))) - elif not isinstance(local_var, np.ndarray): + elif not self.is_numpy_array(local_var): self.add_error_msg("Not supported data type ({t}) for {v} (list and numpy.ndarray)".format( v=var_name, t=type(local_var))) @@ -117,15 +125,12 @@ def check_data_member_int(self, local_var, var_name): self.add_error_msg("{v} is empty (int)".format(v=var_name)) elif isinstance(local_var, list): if isinstance(local_var[0], str) and not self.is_number(local_var[0]): - self.add_error_msg("Not supported data type ({v} string) for {n}[0] (int or float only)".format( + self.add_error_msg("Not supported data type: {n}[0]={v}, string type, not a number (int only)".format( n=var_name, v=local_var[0])) - elif 0 <= str(type(local_var[0])).find('numpy'): - self.log_info("Recommend using numpy instead of python list for {v} ({t}) to avoid side effect".format( - v=var_name, t=type(local_var[0]))) - elif not isinstance(local_var[0], int): + elif 0 > str(type(local_var[0])).find('numpy') and not isinstance(local_var[0], int): self.add_error_msg("Not supported data type ({t}) for {v}[0] (int only)".format( v=var_name, t=type(local_var[0]))) - elif not isinstance(local_var, np.ndarray): + elif not self.is_numpy_array(local_var): self.add_error_msg("Not supported data type ({t}) for {v} (list and numpy.ndarray)".format( v=var_name, t=type(local_var))) @@ -160,6 +165,15 @@ def check_point_data(self): if self.use_var_id: self.check_data_member_string(self.obs_var_table,'obs_var_table') + def convert_to_numpy(self, value_list): + return np.array(value_list) + + def dump(self): + base_met_point_obs.print_point_data(self.get_point_data()) + + def get_count_string(self): + return f' nobs={self.nobs} nhdr={self.nhdr} ntyp={self.nhdr_typ} nsid={self.nhdr_sid} nvld={self.nhdr_vld} nqty={self.nobs_qty} nvar={self.nobs_var}' + def get_point_data(self): if self.nhdr <= 0: self.nhdr = len(self.hdr_lat) @@ -178,34 +192,108 @@ def get_point_data(self): if self.nobs_var <= 0: self.nobs_var = len(self.obs_var_table) self.check_point_data() - return self.__dict__ - def get_type(self, value): - return 'string' if isinstance('str') else type(value) + if not self.is_numpy_array(self.hdr_typ): + self.hdr_typ = self.convert_to_numpy(self.hdr_typ) + if not self.is_numpy_array(self.hdr_sid): + self.hdr_sid = self.convert_to_numpy(self.hdr_sid) + if not self.is_numpy_array(self.hdr_vld): + self.hdr_vld = self.convert_to_numpy(self.hdr_vld) + if not self.is_numpy_array(self.hdr_lat): + self.hdr_lat = self.convert_to_numpy(self.hdr_lat) + if not self.is_numpy_array(self.hdr_lon): + self.hdr_lon = self.convert_to_numpy(self.hdr_lon) + if not self.is_numpy_array(self.hdr_elv): + self.hdr_elv = self.convert_to_numpy(self.hdr_elv) + + if not self.is_numpy_array(self.obs_qty): + self.obs_qty = self.convert_to_numpy(self.obs_qty) + if not self.is_numpy_array(self.obs_hid): + self.obs_hid = self.convert_to_numpy(self.obs_hid) + if not self.is_numpy_array(self.obs_vid): + self.obs_vid = self.convert_to_numpy(self.obs_vid) + if not self.is_numpy_array(self.obs_lvl): + self.obs_lvl = self.convert_to_numpy(self.obs_lvl) + if not self.is_numpy_array(self.obs_hgt): + self.obs_hgt = self.convert_to_numpy(self.obs_hgt) + if not self.is_numpy_array(self.obs_val): + self.obs_val = self.convert_to_numpy(self.obs_val) + + self.count_info = self.get_count_string() + self.met_point_data = self + return self.__dict__ def is_number(self, num_str): return num_str.replace('-','1').replace('+','2').replace('.','3').isdigit() + def is_numpy_array(self, var): + return isinstance(var, np.ndarray) + def log_error_msg(self, err_msg): - print('{p} {m}'.format(p=self.ERROR_P, m=err_msg)) + base_met_point_obs.error_msg(err_msg) def log_error(self, err_msgs): print(self.ERROR_P) for err_line in err_msgs.split('\n'): - print('{p} {m}'.format(p=self.ERROR_P, m=err_line)) + self.log_error_msg(err_line) print(self.ERROR_P) def log_info(self, info_msg): - print('{p} {m}'.format(p=self.INFO_P, m=info_msg)) + base_met_point_obs.info_msg(info_msg) + + def put_data(self, point_obs_dict): + self.hdr_typ = point_obs_dict['hdr_typ'] + self.hdr_sid = point_obs_dict['hdr_sid'] + self.hdr_vld = point_obs_dict['hdr_vld'] + self.hdr_lat = point_obs_dict['hdr_lat'] + self.hdr_lon = point_obs_dict['hdr_lon'] + self.hdr_elv = point_obs_dict['hdr_elv'] + self.hdr_typ_table = point_obs_dict['hdr_typ_table'] + self.hdr_sid_table = point_obs_dict['hdr_sid_table'] + self.hdr_vld_table = point_obs_dict['hdr_vld_table'] + + #Observation data + self.obs_qty = point_obs_dict['obs_qty'] + self.obs_hid = point_obs_dict['obs_hid'] + self.obs_lvl = point_obs_dict['obs_lvl'] + self.obs_hgt = point_obs_dict['obs_hgt'] + self.obs_val = point_obs_dict['obs_val'] + self.obs_vid = point_obs_dict['obs_vid'] + self.obs_var_table = point_obs_dict['obs_var_table'] + self.obs_qty_table = point_obs_dict['obs_qty_table'] + po_array = point_obs_dict.get('obs_unit', None) + if po_array is not None: + self.obs_var_unit = po_array + po_array = point_obs_dict.get('obs_desc', None) + if po_array is not None: + self.obs_var_desc = po_array + + po_array = point_obs_dict.get('hdr_prpt_typ', None) + if po_array is not None: + self.hdr_prpt_typ = po_array + po_array = point_obs_dict.get('hdr_irpt_typ', None) + if po_array is not None: + self.hdr_irpt_typ = po_array + po_array = point_obs_dict.get('hdr_inst_typ', None) + if po_array is not None: + self.hdr_inst_typ = po_array @staticmethod - def is_python_script(arg_value): - return arg_value.startswith(met_point_obs.python_prefix) + def error_msg(msg): + print(f'{get_prompt()} {base_met_point_obs.ERROR_P} {msg}') + + @staticmethod + def info_msg(msg): + print(f'{get_prompt()} {base_met_point_obs.INFO_P} {msg}') @staticmethod def get_python_script(arg_value): return arg_value[len(met_point_obs.python_prefix)+1:] + @staticmethod + def is_python_script(arg_value): + return arg_value.startswith(met_point_obs.python_prefix) + @staticmethod def print_data(key, data_array, show_count=COUNT_SHOW): if isinstance(data_array, list): @@ -274,22 +362,18 @@ def print_point_data(met_point_data, print_subset=True): class csv_point_obs(ABC, base_met_point_obs): def __init__(self, point_data): - super(csv_point_obs, self).__init__() - - hdr_cnt = obs_cnt = len(point_data) self.point_data = point_data - self.nhdr = self.nobs = obs_cnt - self.nhdr_typ = self.nhdr_sid = self.nhdr_vld = hdr_cnt - self.nobs_qty = self.nobs_var = obs_cnt - self.input_file = None - self.ignore_input_file = True + super(csv_point_obs, self).__init__() + self.obs_cnt = obs_cnt = len(point_data) self.obs_qty = [ 0 for _ in range(0, obs_cnt) ] # (nobs_qty) integer, index of self.obs_qty_table self.obs_hid = [ 0 for _ in range(0, obs_cnt) ] # (nobs) integer self.obs_vid = [ 0 for _ in range(0, obs_cnt) ] # (nobs) integer, veriable index from self.obs_var_table or GRIB code - self.obs_lvl = [ nc_tools.FILL_VALUE for _ in range(0, obs_cnt) ] # (nobs) float - self.obs_hgt = [ nc_tools.FILL_VALUE for _ in range(0, obs_cnt) ] # (nobs) float - self.obs_val = [ nc_tools.FILL_VALUE for _ in range(0, obs_cnt) ] # (nobs) float + self.obs_lvl = [ self.FILL_VALUE for _ in range(0, obs_cnt) ] # (nobs) float + self.obs_hgt = [ self.FILL_VALUE for _ in range(0, obs_cnt) ] # (nobs) float + self.obs_val = [ self.FILL_VALUE for _ in range(0, obs_cnt) ] # (nobs) float + + self.convert_point_data() def check_csv_record(self, csv_point_data, index): error_msgs = [] @@ -363,34 +447,36 @@ def convert_point_data(self): obs_var_map = {} obs_qty_map = {} + self.use_var_id = not self.is_grib_code() + index = 0 - # Build map - # names=['typ', 'sid', 'vld', 'lat', 'lon', 'elv', 'var', 'lvl', 'hgt', 'qc', 'obs'] - for csv_point_data in self.point_data: - hdr_typ_str = csv_point_data[0] + #names=['typ', 'sid', 'vld', 'lat', 'lon', 'elv', 'var', 'lvl', 'hgt', 'qc', 'obs'] + for csv_point_record in self.point_data: + # Build header map. + hdr_typ_str = csv_point_record[0] hdr_typ_idx = hdr_typ_map.get(hdr_typ_str,-1) if hdr_typ_idx < 0: hdr_typ_idx = hdr_typ_cnt hdr_typ_map[hdr_typ_str] = hdr_typ_idx hdr_typ_cnt += 1 - hdr_sid_str = csv_point_data[1] + hdr_sid_str = csv_point_record[1] hdr_sid_idx = hdr_sid_map.get(hdr_sid_str,-1) if hdr_sid_idx < 0: hdr_sid_idx = hdr_sid_cnt hdr_sid_map[hdr_sid_str] = hdr_sid_idx hdr_sid_cnt += 1 - hdr_vld_str = csv_point_data[2] + hdr_vld_str = csv_point_record[2] hdr_vld_idx = hdr_vld_map.get(hdr_vld_str,-1) if hdr_vld_idx < 0: hdr_vld_idx = hdr_vld_cnt hdr_vld_map[hdr_vld_str] = hdr_vld_idx hdr_vld_cnt += 1 - lat = csv_point_data[3] - lon = csv_point_data[4] - elv = self.get_num_value(csv_point_data[5] ) + lat = csv_point_record[3] + lon = csv_point_record[4] + elv = self.get_num_value(csv_point_record[5] ) hdr_key = (hdr_typ_idx,hdr_sid_idx,hdr_vld_idx,lat,lon,elv) hdr_idx = hdr_map.get(hdr_key,-1) if hdr_idx < 0: @@ -398,7 +484,7 @@ def convert_point_data(self): hdr_map[hdr_key] = hdr_idx hdr_cnt += 1 - var_id_str = csv_point_data[6] + var_id_str = csv_point_record[6] if self.use_var_id: var_id = obs_var_map.get(var_id_str,-1) if var_id < 0: @@ -408,7 +494,7 @@ def convert_point_data(self): else: var_id = int(var_id_str) - qc_str = csv_point_data[9] + qc_str = csv_point_record[9] qc_id = obs_qty_map.get(qc_str,-1) if qc_id < 0: qc_id = qc_cnt @@ -418,9 +504,9 @@ def convert_point_data(self): # names=['typ', 'sid', 'vld', 'lat', 'lon', 'elv', 'var', 'lvl', 'hgt', 'qc', 'obs'] self.obs_vid[index] = var_id self.obs_hid[index] = hdr_idx - self.obs_lvl[index] = self.get_num_value(csv_point_data[7]) - self.obs_hgt[index] = self.get_num_value(csv_point_data[8]) - self.obs_val[index] = self.get_num_value(csv_point_data[10]) + self.obs_lvl[index] = self.get_num_value(csv_point_record[7]) + self.obs_hgt[index] = self.get_num_value(csv_point_record[8]) + self.obs_val[index] = self.get_num_value(csv_point_record[10]) self.obs_qty[index] = qc_id index += 1 @@ -436,9 +522,9 @@ def convert_point_data(self): self.hdr_typ = [ 0 for _ in range(0, hdr_cnt) ] self.hdr_sid = [ 0 for _ in range(0, hdr_cnt) ] self.hdr_vld = [ 0 for _ in range(0, hdr_cnt) ] - self.hdr_lat = [ nc_tools.FILL_VALUE for _ in range(0, hdr_cnt) ] - self.hdr_lon = [ nc_tools.FILL_VALUE for _ in range(0, hdr_cnt) ] - self.hdr_elv = [ nc_tools.FILL_VALUE for _ in range(0, hdr_cnt) ] + self.hdr_lat = [ self.FILL_VALUE for _ in range(0, hdr_cnt) ] + self.hdr_lon = [ self.FILL_VALUE for _ in range(0, hdr_cnt) ] + self.hdr_elv = [ self.FILL_VALUE for _ in range(0, hdr_cnt) ] for key, idx in hdr_map.items(): self.hdr_typ[idx] = key[0] self.hdr_sid[idx] = key[1] @@ -463,16 +549,15 @@ def convert_point_data(self): for key, idx in obs_var_map.items(): self.obs_var_table[idx] = key - return self.get_point_data() - def get_num_value(self, column_value): num_value = column_value if isinstance(column_value, str): - if column_value.lower() == 'na': - num_value = nc_tools.FILL_VALUE - elif self.is_number(column_value): + if self.is_number(column_value): num_value = float(column_value) - num_value = nc_tools.FILL_VALUE + else: + num_value = self.FILL_VALUE + if column_value.lower() != 'na' and column_value.lower() != 'n/a': + self.log_info(f'{column_value} is not a number, converted to the missing value') return num_value def is_grib_code(self): @@ -488,7 +573,7 @@ def is_grib_code(self): def is_num_string(self, column_value): is_string = isinstance(column_value, str) if is_string: - is_num = True if self.is_number(column_value) or column_value.lower() == 'na' else False + is_num = True if self.is_number(column_value) or column_value.lower() == 'na' or column_value.lower() == 'n/a' else False else: is_num = True return is_string, is_num @@ -496,6 +581,8 @@ def is_num_string(self, column_value): class met_point_obs(ABC, base_met_point_obs): + MET_ENV_RUN = 'MET_FORCE_TO_RUN' + @abstractmethod def read_data(self, args): # args can be input_file_name, list, or dictionary @@ -514,31 +601,13 @@ def read_data(self, args): ''' pass - -# Note: caller should import netCDF4 -# the argements nc_group(dataset) and nc_var should not be None -class nc_tools(): - - FILL_VALUE = -9999. - met_missing = -99999999. - - @staticmethod - def get_num_array(nc_group, var_name): - nc_var = nc_group.variables.get(var_name, None) - #return [] if nc_var is None else nc_var[:].filled(nc_var._FillValue if '_FillValue' in nc_var.ncattrs() else nc_tools.met_missing) - return [] if nc_var is None else nc_var[:] - @staticmethod - def get_ncbyte_array_to_str(nc_var): - nc_str_data = nc_var[:] - if nc_var.datatype.name == 'bytes8': - nc_str_data = [ str(s.compressed(),"utf-8") for s in nc_var[:] ] - return nc_str_data + def get_prompt(): + return get_prompt() @staticmethod - def get_string_array(nc_group, var_name): - nc_var = nc_group.variables.get(var_name, None) - return [] if nc_var is None else nc_tools.get_ncbyte_array_to_str(nc_var) + def is_python_prefix(user_cmd): + return user_cmd.startswith(base_met_point_obs.python_prefix) # This is a sample drived class @@ -568,12 +637,15 @@ def read_data(self, arg_map={}): self.obs_var_table = [ "TMP", "RH" ] self.obs_qty_table = [ "NA" ] -def convert_point_data(point_data, check_all_records=False): - _csv_point_data = csv_point_obs(point_data) - if _csv_point_data.is_grib_code(): - _csv_point_data.use_var_id = False - _csv_point_data.check_csv_point_data(check_all_records) - return _csv_point_data.convert_point_data() +def convert_point_data(point_data, check_all_records=False, input_type='csv'): + tmp_point_data = {} + if 'csv' == input_type: + csv_point_data = csv_point_obs(point_data) + csv_point_data.check_csv_point_data(check_all_records) + tmp_point_data = csv_point_data.get_point_data() + else: + base_met_point_obs.error_msg('Not supported input type: {input_type}') + return tmp_point_data def main(): args = {} # or args = [] diff --git a/scripts/python/met_point_obs_nc.py b/scripts/python/met_point_obs_nc.py new file mode 100644 index 0000000000..cb86be5011 --- /dev/null +++ b/scripts/python/met_point_obs_nc.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 + +''' +Separated from read_met_point_obs on Feb 09, 2023 + +@author: hsoh + +This script reads the MET point observation NetCDF file like MET tools do. +''' + +import os +import sys +from datetime import datetime +import numpy as np +import netCDF4 as nc + +from met_point_obs import met_point_obs, base_met_point_obs, get_prompt + +DO_PRINT_DATA = False +ARG_PRINT_DATA = 'show_data' + +# Note: caller should import netCDF4 +# the argements nc_group(dataset) and nc_var should not be None +class nc_tools(): + + met_missing = -99999999. + + @staticmethod + def get_num_array(nc_group, var_name): + nc_var = nc_group.variables.get(var_name, None) + return [] if nc_var is None else nc_var[:] + + @staticmethod + def get_ncbyte_array_to_str(nc_var): + nc_str_data = nc_var[:] + if nc_var.datatype.name == 'bytes8': + nc_str_data = [ str(s.compressed(),"utf-8") for s in nc_var[:] ] + return nc_str_data + + @staticmethod + def get_string_array(nc_group, var_name): + nc_var = nc_group.variables.get(var_name, None) + return [] if nc_var is None else nc_tools.get_ncbyte_array_to_str(nc_var) + + +class nc_point_obs(met_point_obs): + + # args should be string, list, or dictionary + def get_nc_filename(self, args): + nc_filename = None + if isinstance(args, dict): + nc_filename = args.get('nc_name',None) + elif isinstance(args, list): + nc_filename = args[0] + elif args != ARG_PRINT_DATA: + nc_filename = args + + return nc_filename + + def read_data(self, nc_filename): + if nc_filename is None: + self.log_error_msg("The input NetCDF filename is missing") + elif not os.path.exists(nc_filename): + self.log_error_msg(f"input NetCDF file ({nc_filename}) does not exist") + else: + dataset = nc.Dataset(nc_filename, 'r') + # Header + self.hdr_typ = dataset['hdr_typ'][:] + self.hdr_sid = dataset['hdr_sid'][:] + self.hdr_vld = dataset['hdr_vld'][:] + self.hdr_lat = dataset['hdr_lat'][:] + self.hdr_lon = dataset['hdr_lon'][:] + self.hdr_elv = dataset['hdr_elv'][:] + self.hdr_typ_table = nc_tools.get_string_array(dataset, 'hdr_typ_table') + self.hdr_sid_table = nc_tools.get_string_array(dataset, 'hdr_sid_table') + self.hdr_vld_table = nc_tools.get_string_array(dataset, 'hdr_vld_table') + + nc_var = dataset.variables.get('obs_unit', None) + if nc_var: + self.obs_var_unit = nc_var[:] + nc_var = dataset.variables.get('obs_desc', None) + if nc_var: + self.obs_var_desc = nc_var[:] + + nc_var = dataset.variables.get('hdr_prpt_typ', None) + if nc_var: + self.hdr_prpt_typ = nc_var[:] + nc_var = dataset.variables.get('hdr_irpt_typ', None) + if nc_var: + self.hdr_irpt_typ = nc_var[:] + nc_var = dataset.variables.get('hdr_inst_typ', None) + if nc_var: + self.hdr_inst_typ =nc_var[:] + + #Observation data + self.hdr_sid = dataset['hdr_sid'][:] + self.obs_qty = np.array(dataset['obs_qty'][:]) + self.obs_hid = np.array(dataset['obs_hid'][:]) + self.obs_lvl = np.array(dataset['obs_lvl'][:]) + self.obs_hgt = np.array(dataset['obs_hgt'][:]) + self.obs_val = np.array(dataset['obs_val'][:]) + nc_var = dataset.variables.get('obs_vid', None) + if nc_var is None: + self.use_var_id = False + nc_var = dataset.variables.get('obs_gc', None) + else: + self.obs_var_table = nc_tools.get_string_array(dataset, 'obs_var') + if nc_var: + self.obs_vid = np.array(nc_var[:]) + + self.obs_qty_table = nc_tools.get_string_array(dataset, 'obs_qty_table') + + def save_ncfile(self, nc_filename): + met_data = self.get_point_data() + with nc.Dataset(nc_filename, 'w') as nc_dataset: + self.set_nc_data(nc_dataset) + return met_data + + def set_nc_data(self, nc_dataset): + return nc_point_obs.write_nc_data(nc_dataset, self) + + @staticmethod + def write_nc_file(nc_filename, point_obs): + with nc.Dataset(nc_filename, 'w') as nc_dataset: + nc_point_obs.set_nc_data(nc_dataset, point_obs) + + @staticmethod + def write_nc_data(nc_dataset, point_obs): + do_nothing = False + if 0 == point_obs.nhdr: + do_nothing = True + base_met_point_obs.info_msg("the header is empty") + if 0 == point_obs.nobs: + do_nothing = True + base_met_point_obs.info_msg("the observation data is empty") + if do_nothing: + print() + return + + # Set global attributes + nc_dataset.MET_Obs_version = "1.02" ; + nc_dataset.use_var_id = "true" if point_obs.use_var_id else "false" + + # Create dimensions + nc_dataset.createDimension('mxstr', 16) + nc_dataset.createDimension('mxstr2', 40) + nc_dataset.createDimension('mxstr3', 80) + nc_dataset.createDimension('nhdr', point_obs.nhdr) + nc_dataset.createDimension('nobs', point_obs.nobs) + #npbhdr = len(point_obs.hdr_prpt_typ) + if 0 < point_obs.npbhdr: + nc_dataset.createDimension('npbhdr', point_obs.npbhdr) + nc_dataset.createDimension('nhdr_typ', point_obs.nhdr_typ) + nc_dataset.createDimension('nhdr_sid', point_obs.nhdr_sid) + nc_dataset.createDimension('nhdr_vld', point_obs.nhdr_vld) + nc_dataset.createDimension('nobs_qty', point_obs.nobs_qty) + nc_dataset.createDimension('obs_var_num', point_obs.nobs_var) + + type_for_string = 'S1' # np.byte + dims_hdr = ('nhdr',) + dims_obs = ('nobs',) + + # Create header and observation variables + var_hdr_typ = nc_dataset.createVariable('hdr_typ', np.int32, dims_hdr, fill_value=-9999) + var_hdr_sid = nc_dataset.createVariable('hdr_sid', np.int32, dims_hdr, fill_value=-9999) + var_hdr_vld = nc_dataset.createVariable('hdr_vld', np.int32, dims_hdr, fill_value=-9999) + var_hdr_lat = nc_dataset.createVariable('hdr_lat', np.float32, dims_hdr, fill_value=-9999.) + var_hdr_lon = nc_dataset.createVariable('hdr_lon', np.float32, dims_hdr, fill_value=-9999.) + var_hdr_elv = nc_dataset.createVariable('hdr_elv', np.float32, dims_hdr, fill_value=-9999.) + + var_obs_qty = nc_dataset.createVariable('obs_qty', np.int32, dims_obs, fill_value=-9999) + var_obs_hid = nc_dataset.createVariable('obs_hid', np.int32, dims_obs, fill_value=-9999) + var_obs_vid = nc_dataset.createVariable('obs_vid', np.int32, dims_obs, fill_value=-9999) + var_obs_lvl = nc_dataset.createVariable('obs_lvl', np.float32, dims_obs, fill_value=-9999.) + var_obs_hgt = nc_dataset.createVariable('obs_hgt', np.float32, dims_obs, fill_value=-9999.) + var_obs_val = nc_dataset.createVariable('obs_val', np.float32, dims_obs, fill_value=-9999.) + + if 0 == point_obs.npbhdr: + var_hdr_prpt_typ = None + var_hdr_irpt_typ = None + var_hdr_inst_typ = None + else: + dims_npbhdr = ('npbhdr',) + var_hdr_prpt_typ = nc_dataset.createVariable('hdr_prpt_typ', np.int32, dims_npbhdr, fill_value=-9999.) + var_hdr_irpt_typ = nc_dataset.createVariable('hdr_irpt_typ', np.int32, dims_npbhdr, fill_value=-9999.) + var_hdr_inst_typ = nc_dataset.createVariable('hdr_inst_typ', np.int32, dims_npbhdr, fill_value=-9999.) + + var_hdr_typ_table = nc_dataset.createVariable('hdr_typ_table', type_for_string, ('nhdr_typ','mxstr2')) + var_hdr_sid_table = nc_dataset.createVariable('hdr_sid_table', type_for_string, ('nhdr_sid','mxstr2')) + var_hdr_vld_table = nc_dataset.createVariable('hdr_vld_table', type_for_string, ('nhdr_vld','mxstr')) + var_obs_qty_table = nc_dataset.createVariable('obs_qty_table', type_for_string, ('nobs_qty','mxstr')) + var_obs_var_table = nc_dataset.createVariable('obs_var', type_for_string, ('obs_var_num','mxstr2')) + var_obs_var_unit = nc_dataset.createVariable('obs_unit', type_for_string, ('obs_var_num','mxstr2')) + var_obs_var_desc = nc_dataset.createVariable('obs_desc', type_for_string, ('obs_var_num','mxstr3')) + + # Set variables + var_hdr_typ[:] = point_obs.hdr_typ[:] + var_hdr_sid[:] = point_obs.hdr_sid[:] + var_hdr_vld[:] = point_obs.hdr_vld[:] + var_hdr_lat[:] = point_obs.hdr_lat[:] + var_hdr_lon[:] = point_obs.hdr_lon[:] + var_hdr_elv[:] = point_obs.hdr_elv[:] + for i in range(0, point_obs.nhdr_typ): + for j in range(0, len(point_obs.hdr_typ_table[i])): + var_hdr_typ_table[i,j] = point_obs.hdr_typ_table[i][j] + for i in range(0, point_obs.nhdr_sid): + for j in range(0, len(point_obs.hdr_sid_table[i])): + var_hdr_sid_table[i,j] = point_obs.hdr_sid_table[i][j] + for i in range(0, point_obs.nhdr_vld): + for j in range(0, len(point_obs.hdr_vld_table[i])): + var_hdr_vld_table[i,j] = point_obs.hdr_vld_table[i][j] + if 0 < point_obs.npbhdr: + var_hdr_prpt_typ[:] = point_obs.hdr_prpt_typ[:] + var_hdr_irpt_typ[:] = point_obs.hdr_irpt_typ[:] + var_hdr_inst_typ[:] = point_obs.hdr_inst_typ[:] + + var_obs_qty[:] = point_obs.obs_qty[:] + var_obs_hid[:] = point_obs.obs_hid[:] + var_obs_vid[:] = point_obs.obs_vid[:] + var_obs_lvl[:] = point_obs.obs_lvl[:] + var_obs_hgt[:] = point_obs.obs_hgt[:] + var_obs_val[:] = point_obs.obs_val[:] + for i in range(0, point_obs.nobs_var): + for j in range(0, len(point_obs.obs_var_table[i])): + var_obs_var_table[i,j] = point_obs.obs_var_table[i][j] + var_obs_var_unit[i] = "" if i >= len(point_obs.obs_var_unit) else point_obs.obs_var_unit[i] + var_obs_var_desc[i] = "" if i >= len(point_obs.obs_var_desc) else point_obs.obs_var_desc[i] + for i in range(0, point_obs.nobs_qty): + for j in range(0, len(point_obs.obs_qty_table[i])): + var_obs_qty_table[i,j] = point_obs.obs_qty_table[i][j] + + # Set variable attributes + var_hdr_typ.long_name = "index of message type" + var_hdr_sid.long_name = "index of station identification" + var_hdr_vld.long_name = "index of valid time" + var_hdr_lat.long_name = "latitude" + var_hdr_lat.units = "degrees_north" + var_hdr_lon.long_name = "longitude" + var_hdr_lon.units = "degrees_east" + var_hdr_elv.long_name = "elevation" + var_hdr_elv.units = "meters above sea level (msl)" + + var_obs_qty.long_name = "index of quality flag" + var_obs_hid.long_name = "index of matching header data" + var_obs_vid.long_name = "index of BUFR variable corresponding to the observation type" + var_obs_lvl.long_name = "pressure level (hPa) or accumulation interval (sec)" + var_obs_hgt.long_name = "height in meters above sea level (msl)" + var_obs_val.long_name = "observation value" + var_hdr_typ_table.long_name = "message type" + var_hdr_sid_table.long_name = "station identification" + var_hdr_vld_table.long_name = "valid time" + var_hdr_vld_table.units = "YYYYMMDD_HHMMSS UTC" + var_obs_qty_table.long_name = "quality flag" + var_obs_var_table.long_name = "variable names" + var_obs_var_unit.long_name = "variable units" + var_obs_var_desc.long_name = "variable descriptions" + + +def main(argv): + if len(argv) != 1 and argv[1] != ARG_PRINT_DATA: + netcdf_filename = argv[1] + tmp_nc_name = 'tmp_met_point.nc' + point_obs_data = nc_point_obs() + point_obs_data.read_data(point_obs_data.get_nc_filename(netcdf_filename)) + met_point_data = point_obs_data.save_ncfile(tmp_nc_name) + print(f'{get_prompt()} saved met_point_data to {tmp_nc_name}') + met_point_data['met_point_data'] = point_obs_data + + if DO_PRINT_DATA or ARG_PRINT_DATA == argv[-1]: + met_point_obs.print_point_data(met_point_data) + +if __name__ == '__main__': + start_time = datetime.now() + main(sys.argv) + run_time = datetime.now() - start_time + print(f'{get_prompt()} Done python script {sys.argv[0]} took {run_time}') diff --git a/scripts/python/read_met_point_obs.py b/scripts/python/read_met_point_obs.py index 60f2c62065..57ccd22e7a 100755 --- a/scripts/python/read_met_point_obs.py +++ b/scripts/python/read_met_point_obs.py @@ -1,121 +1,91 @@ +#!/usr/bin/env python3 ''' Created on Nov 10, 2021 @author: hsoh This script reads the MET point observation NetCDF file like MET tools do. + +Usage: + + python3 read_met_point_obs.py + python3 read_met_point_obs.py + : 11 columns + 'typ', 'sid', 'vld', 'lat', 'lon', 'elv', 'var', 'lvl', 'hgt', 'qc', 'obs' + string columns: 'typ', 'sid', 'vld', 'var', , 'qc' + numeric columns 'lat', 'lon', 'elv', 'lvl', 'hgt', 'qc', 'obs' + python3 read_met_point_obs.py + ''' import os import sys from datetime import datetime -import numpy as np -import netCDF4 as nc + +met_base_dir = os.getenv('MET_BASE',None) +if met_base_dir is not None: + sys.path.append(os.path.join(met_base_dir, 'python')) from met_point_obs import met_point_obs, sample_met_point_obs +from met_point_obs_nc import nc_point_obs DO_PRINT_DATA = False ARG_PRINT_DATA = 'show_data' -class nc_point_obs(met_point_obs): - - def read_data(self, args): - # args should be list or dictionaryu - if isinstance(args, dict): - nc_filename = args.get('nc_name',None) - elif isinstance(args, list): - nc_filename = args[0] - else: - nc_filename = args - if nc_filename is None: - print("==ERROR== The input NetCDF filename is missing") - elif not os.path.exists(nc_filename): - print("==ERROR== input NetCDF file ({f}) does not exist".format(f=nc_filename)) - else: - dataset = nc.Dataset(nc_filename, 'r') - # Header - self.hdr_typ = dataset['hdr_typ'][:] - self.hdr_sid = dataset['hdr_sid'][:] - self.hdr_vld = dataset['hdr_vld'][:] - self.hdr_lat = dataset['hdr_lat'][:] - self.hdr_lon = dataset['hdr_lon'][:] - self.hdr_elv = dataset['hdr_elv'][:] - self.hdr_typ_table = nc_tools.get_string_array(dataset, 'hdr_typ_table') - self.hdr_sid_table = nc_tools.get_string_array(dataset, 'hdr_sid_table') - self.hdr_vld_table = nc_tools.get_string_array(dataset, 'hdr_vld_table') - - nc_var = dataset.variables.get('hdr_prpt_typ', None) - if nc_var: - self.hdr_prpt_typ = nc_var[:] - nc_var = dataset.variables.get('hdr_irpt_typ', None) - if nc_var: - self.hdr_irpt_typ = nc_var[:] - nc_var = dataset.variables.get('hdr_inst_typ', None) - if nc_var: - self.hdr_inst_typ =nc_var[:] - - #Observation data - self.hdr_sid = dataset['hdr_sid'][:] - self.obs_qty = np.array(dataset['obs_qty'][:]) - self.obs_hid = np.array(dataset['obs_hid'][:]) - self.obs_lvl = np.array(dataset['obs_lvl'][:]) - self.obs_hgt = np.array(dataset['obs_hgt'][:]) - self.obs_val = np.array(dataset['obs_val'][:]) - nc_var = dataset.variables.get('obs_vid', None) - if nc_var is None: - self.use_var_id = False - nc_var = dataset.variables.get('obs_gc', None) - else: - self.obs_var_table = nc_tools.get_string_array(dataset, 'obs_var') - if nc_var: - self.obs_vid = np.array(nc_var[:]) - - self.obs_qty_table = nc_tools.get_string_array(dataset, 'obs_qty_table') - - -class nc_tools(): - - @staticmethod - def get_num_array(dataset, var_name): - nc_var = dataset.variables.get(var_name, None) - return nc_var[:].filled(nc_var._FillValue if '_FillValue' in nc_var.ncattrs() else -9999) if nc_var else [] - - @staticmethod - def get_ncbyte_array_to_str(nc_var): - nc_str_data = nc_var[:] - if nc_var.datatype.name == 'bytes8': - nc_str_data = [ str(s.compressed(),"utf-8") for s in nc_var[:]] - return nc_str_data - - @staticmethod - def get_string_array(dataset, var_name): - nc_var = dataset.variables.get(var_name, None) - return nc_tools.get_ncbyte_array_to_str(nc_var) if nc_var else [] - - start_time = datetime.now() +prompt = met_point_obs.get_prompt() point_obs_data = None -if len(sys.argv) == 1: +if len(sys.argv) == 1 or ARG_PRINT_DATA == sys.argv[1]: point_obs_data = sample_met_point_obs() point_obs_data.read_data([]) +elif met_point_obs.is_python_prefix(sys.argv[1]): + import importlib.util + + print("{p} Python Script:\t".format(p=prompt) + repr(sys.argv[0])) + print("{p} User Command:\t".format(p=prompt) + repr(' '.join(sys.argv[2:]))) + + pyembed_module_name = sys.argv[2] + sys.argv = sys.argv[1:] + + # append user script dir to system path + pyembed_dir, pyembed_file = os.path.split(pyembed_module_name) + if pyembed_dir: + sys.path.insert(0, pyembed_dir) + + if not pyembed_module_name.endswith('.py'): + pyembed_module_name += '.py' + os.environ[met_point_obs.MET_ENV_RUN] = "TRUE" + + user_base = os.path.basename(pyembed_module_name).replace('.py','') + + spec = importlib.util.spec_from_file_location(user_base, pyembed_module_name) + met_in = importlib.util.module_from_spec(spec) + spec.loader.exec_module(met_in) + + met_point_obs = met_in.met_point_obs + print("met_point_obs: ", met_point_obs) + met_point_data = met_in.met_point_data + print("met_point_data: ", met_point_data) + #print(hasattr("met_in: ", dir(met_in))) + #met_point_data = met_point_obs.get_point_data() + #met_point_data = None if met_in.get('met_point_data', None) else met_in.met_point_data + #met_data = None if met_in.get('met_data', None) else met_in.met_data + print(met_point_data) else: netcdf_filename = sys.argv[1] args = [ netcdf_filename ] #args = { 'nc_name': netcdf_filename } point_obs_data = nc_point_obs() - point_obs_data.read_data(args) - - if ARG_PRINT_DATA == sys.argv[-1]: - DO_PRINT_DATA = True + point_obs_data.read_data(point_obs_data.get_nc_filename(args)) if point_obs_data is not None: met_point_data = point_obs_data.get_point_data() met_point_data['met_point_data'] = point_obs_data - if DO_PRINT_DATA: - met_point_obs.print_point_data(met_point_data) + if DO_PRINT_DATA or ARG_PRINT_DATA == sys.argv[-1]: + point_obs_data.dump() run_time = datetime.now() - start_time -print('Done python script {s} took {t}'.format(s=sys.argv[0], t=run_time)) +print('{p} Done python script {s} took {t}'.format(p=prompt, s=sys.argv[0], t=run_time)) diff --git a/src/basic/vx_math/pwl.cc b/src/basic/vx_math/pwl.cc index eef083a600..411a1f734b 100644 --- a/src/basic/vx_math/pwl.cc +++ b/src/basic/vx_math/pwl.cc @@ -57,12 +57,7 @@ PiecewiseLinear::~PiecewiseLinear() { -n_alloc = N = 0; - -if ( X ) { delete [] X; X = (double *) 0; } -if ( Y ) { delete [] Y; Y = (double *) 0; } - - if ( Name != "" ) { Name.clear(); } +clear(); } @@ -100,34 +95,32 @@ return ( *this ); //////////////////////////////////////////////////////////////////////// -void PiecewiseLinear::init_from_scratch() +PiecewiseLinear::PiecewiseLinear(const char *NAME, int Npoints, const double *XX, const double *YY) { -n_alloc = N = 0; +init_from_scratch(); -Name.clear(); +Name = NAME; -X = Y = (double *) 0; +for (int j=0; j memory allocation error\n\n"; - exit ( 1 ); +out << prefix << "PiecewiseLinear ... \n"; -} +out << prefix << " Name = " << Name << "\n"; -memset(u, 0, n*sizeof(double)); -memset(v, 0, n*sizeof(double)); +out << prefix << " # Points = " << N << "\n"; - // - // copy any existing information over - // -for (k=0; k 0 ) { + out.width(10); out.precision(3); out << (Y[j]); - delete [] X; X = (double *) 0; - delete [] Y; Y = (double *) 0; + out << "\n"; } - // - // swap - // - -X = u; u = (double *) 0; -Y = v; v = (double *) 0; - - // - // done - // - -n_alloc = n; +out.flush(); return; @@ -253,26 +208,13 @@ return; //////////////////////////////////////////////////////////////////////// -PiecewiseLinear::PiecewiseLinear(const char *NAME, int Npoints, const double *XX, const double *YY) +void PiecewiseLinear::set_name(const ConcatString NAME) { -init_from_scratch(); - - if ( NAME ) set_name((string)NAME); - -extend(Npoints); +Name = NAME; -int j; - -for (j=0; j= N) ) { +if ( k < 0 || k >= X.size() ) { mlog << Error << "\nPiecewiseLinear::x(int) -> range check error\n\n"; @@ -304,7 +246,7 @@ double PiecewiseLinear::y(int k) const { -if ( (k < 0) || (k >= N) ) { +if ( k < 0 || k >= Y.size() ) { mlog << Error << "\nPiecewiseLinear::y(int) -> range check error\n\n"; @@ -320,10 +262,27 @@ return ( Y[k] ); //////////////////////////////////////////////////////////////////////// +void PiecewiseLinear::add_point(double xx, double yy) + +{ + +X.push_back(xx); +Y.push_back(yy); + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + + double PiecewiseLinear::operator()(double t) const { +int N = X.size(); + if ( t < X[0] ) return ( Y[0] ); if ( t > X[N - 1] ) return ( Y[N - 1] ); @@ -340,7 +299,6 @@ for (j=0; j<(N - 1); ++j) { } - // // flow of control should not reach this point // @@ -355,87 +313,6 @@ return ( -100 ); // just to satisfy the compiler } -//////////////////////////////////////////////////////////////////////// - - -void PiecewiseLinear::set_name(const ConcatString T) - -{ - - if ( Name != "" ) { Name.clear(); } - - - Name = T; - - - return; - -} - - -//////////////////////////////////////////////////////////////////////// - - -void PiecewiseLinear::add_point(double xx, double yy) - -{ - -extend(N + 1); - -X[N] = xx; -Y[N] = yy; - -++N; - - -return; - -} - - -//////////////////////////////////////////////////////////////////////// - - -void PiecewiseLinear::dump(ostream & out, int indent_depth) const - -{ - -int j; -Indent prefix; - - -prefix.depth = indent_depth; - - -out << prefix << "PiecewiseLinear ... \n"; - -out << prefix << " Name = " << Name << "\n"; - -out << prefix << " # Points = " << N << "\n"; - - -out.setf(ios::fixed); - -for (j=0; j= 0) && (x[j] > x_in) ) --j; - - // - // range check - // - -if ( j < 0 ) { y_out = y[0]; return ( 0 ); } - -if ( j >= (n - 1) ) { y_out = y[n - 1]; return ( 0 ); } - - // - // interpolate - // - -if ( is_eq(x[j + 1], x[j]) ) { - - y_out = y[j]; - -} else { - - m = (y[j + 1] - y[j])/(x[j + 1] - x[j]); - - y_out = y[j] + m*(x_in - x[j]); - -} - - // - // done - // - -return ( 1 ); - -} - - -//////////////////////////////////////////////////////////////////////// - - - diff --git a/src/basic/vx_math/pwl.h b/src/basic/vx_math/pwl.h index 410fe66993..eb4d64dfea 100644 --- a/src/basic/vx_math/pwl.h +++ b/src/basic/vx_math/pwl.h @@ -18,7 +18,7 @@ #include - +#include //////////////////////////////////////////////////////////////////////// @@ -39,17 +39,10 @@ class PiecewiseLinear { protected: - void extend(int); - - int N; - - int n_alloc; - ConcatString Name; - double * X; - double * Y; - + std::vector X; + std::vector Y; public: @@ -95,7 +88,7 @@ class PiecewiseLinear { //////////////////////////////////////////////////////////////////////// -inline int PiecewiseLinear::n_points() const { return ( N ); } +inline int PiecewiseLinear::n_points() const { return ( X.size() ); } inline const char * PiecewiseLinear::name() const { return ( Name.c_str() ); } @@ -103,12 +96,6 @@ inline const char * PiecewiseLinear::name() const { return ( Name.c_str() ); } //////////////////////////////////////////////////////////////////////// -extern int pwl_interpolate(const double * y, const double * x, int n, double x_in, double & y_out); - - -//////////////////////////////////////////////////////////////////////// - - #endif /* __PIECEWISE_LINEAR_H__ */ diff --git a/src/basic/vx_util/ascii_table.cc b/src/basic/vx_util/ascii_table.cc index 343a5aad23..988b4bdc8f 100644 --- a/src/basic/vx_util/ascii_table.cc +++ b/src/basic/vx_util/ascii_table.cc @@ -19,6 +19,7 @@ using namespace std; #include #include #include +#include #include "vx_log.h" #include "vx_cal.h" @@ -1336,21 +1337,14 @@ if ( Nrows < 0 || Nrows >= INT_MAX ) { if ( Nrows <= 2 ) return; -int * left = new int [Nrows]; -int * right = new int [Nrows]; +std::vector left (Nrows, 0); +std::vector right(Nrows, 0); int r, c, n, k; int max_left, max_right; const char fill_char = ' '; const int r_start = 1; // skip the header row - -for (r=0; r +#include #include "num_array.h" #include "int_array.h" @@ -34,27 +35,6 @@ static const int crc_array_alloc_inc = 25; -//////////////////////////////////////////////////////////////////////// - - -template - -int is_bigger(const void * p1, const void * p2) - -{ - -const T & a = *((T *) p1); -const T & b = *((T *) p2); - -if ( a < b ) return ( -1 ); -if ( a > b ) return ( 1 ); - -return ( 0 ); - -} - - - //////////////////////////////////////////////////////////////////////// @@ -69,7 +49,7 @@ class CRC_Array { void assign(const CRC_Array &); - T * e; + std::vector e; int Nelements; @@ -144,12 +124,6 @@ class CRC_Array { }; -//////////////////////////////////////////////////////////////////////// - - -// static int is_bigger(const void *, const void *); - - //////////////////////////////////////////////////////////////////////// @@ -205,8 +179,6 @@ void CRC_Array::init_from_scratch() { -e = (T *) 0; - clear(); return; @@ -223,7 +195,7 @@ void CRC_Array::clear() { -if ( e ) { delete [] e; e = (T *) 0; } +e.clear(); Nelements = Nalloc = 0; @@ -247,13 +219,7 @@ if ( a.Nelements == 0 ) return; extend(a.Nelements); -int j; - -for (j=0; j<(a.Nelements); ++j) { - - e[j] = a.e[j]; - -} +e = a.e; Nelements = a.Nelements; @@ -286,36 +252,7 @@ if ( ! exact ) { } -T * u = (T *) 0; - -u = new T [len]; - -if ( !u ) { - - mlog << Error << "\nvoid CRC_Array::extend(int, bool) -> " - << "memory allocation error\n\n"; - - exit ( 1 ); - -} - -int j; - -memset(u, 0, len*sizeof(T)); - -if ( e ) { - - for (j=0; j::add(const T & k) extend(Nelements + 1, false); -e[Nelements++] = k; +e.push_back(k); + +Nelements++; return; @@ -555,7 +494,9 @@ int j; for (j=0; j<(a.Nelements); ++j) { - e[Nelements++] = a.e[j]; + e.push_back(a.e[j]); + + Nelements++; } @@ -604,7 +545,7 @@ void CRC_Array::sort_increasing() if ( Nelements <= 1 ) return; -qsort(e, Nelements, sizeof(*e), is_bigger); +std::sort(e.begin(), e.end()); return; diff --git a/src/basic/vx_util/util_constants.h b/src/basic/vx_util/util_constants.h index 05a5996685..e7d86d38ad 100644 --- a/src/basic/vx_util/util_constants.h +++ b/src/basic/vx_util/util_constants.h @@ -18,6 +18,7 @@ //////////////////////////////////////////////////////////////////////// // Released versions of MET +static const char met_version_11_0_1[] = "V11.0.1"; static const char met_version_11_0_0[] = "V11.0.0"; static const char met_version_10_1_0[] = "V10.1.0"; static const char met_version_10_0_0[] = "V10.0.0"; @@ -41,7 +42,7 @@ static const char met_version_1_1[] = "V1.1"; //////////////////////////////////////////////////////////////////////// -static const char * const met_version = met_version_11_0_0; +static const char * const met_version = met_version_11_0_1; static const char default_met_data_dir[] = "MET_BASE"; static const char txt_file_ext[] = ".txt"; static const char stat_file_ext[] = ".stat"; diff --git a/src/libcode/vx_data2d/table_lookup.cc b/src/libcode/vx_data2d/table_lookup.cc index dbc920ae21..d9c4605d9e 100644 --- a/src/libcode/vx_data2d/table_lookup.cc +++ b/src/libcode/vx_data2d/table_lookup.cc @@ -617,9 +617,6 @@ void TableFlatFile::init_from_scratch() { -g1e = (Grib1TableEntry **) 0; -g2e = (Grib2TableEntry **) 0; - clear(); } @@ -634,30 +631,8 @@ void TableFlatFile::clear() int j; -if ( g1e ) { - - for (j=0; jdump(out, depth + 1); + g1e[j].dump(out, depth + 1); } @@ -696,7 +671,7 @@ for (j=0; jdump(out, depth + 1); + g2e[j].dump(out, depth + 1); } @@ -722,15 +697,7 @@ if ( f.N_grib1_elements != 0 ) { N_grib1_elements = N_grib1_alloc = f.N_grib1_elements; - g1e = new Grib1TableEntry * [N_grib1_elements]; - - for (j=0; j 0 ) { - - for (j=0; j 0 ) { - - for (j=0; jparse_line(line.c_str()); + status = e.parse_line(line.c_str()); if ( ! status ) { @@ -957,9 +873,11 @@ while ( line.read_line(in) ) { } // - // increment counter + // store entry and increment counter // + g1e.push_back(e); + j++; } // while @@ -988,6 +906,7 @@ bool TableFlatFile::read_grib2(istream & in, const char * filename, const int n) int j; ConcatString line; +Grib2TableEntry e; bool status = false; // @@ -1016,9 +935,7 @@ while ( line.read_line(in) ) { line << "\n"; - g2e[N_grib2_elements + j] = new Grib2TableEntry; - - status = g2e[N_grib2_elements + j]->parse_line(line.c_str()); + status = e.parse_line(line.c_str()); if ( ! status ) { @@ -1031,9 +948,11 @@ while ( line.read_line(in) ) { } // - // increment counter + // store entry and increment counter // + g2e.push_back(e); + j++; } // while @@ -1062,9 +981,9 @@ e.clear(); for (j=0; jcode == code) && (g1e[j]->table_number == table_number) ) { + if ( (g1e[j].code == code) && (g1e[j].table_number == table_number) ) { - e = *(g1e[j]); + e = g1e[j]; return ( true ); @@ -1091,14 +1010,14 @@ bool TableFlatFile::lookup_grib1(int code, int table_number, int center, int sub for (j=0; jsubcenter == -1){ + if( g1e[j].subcenter == -1){ matching_subsenter = -1; } - if ( (g1e[j]->code == code) && (g1e[j]->table_number == table_number) - && (g1e[j]->center == center) && (g1e[j]->subcenter == matching_subsenter)) { + if ( (g1e[j].code == code) && (g1e[j].table_number == table_number) && + (g1e[j].center == center) && (g1e[j].subcenter == matching_subsenter)) { - e = *(g1e[j]); + e = g1e[j]; return ( true ); @@ -1138,15 +1057,15 @@ bool TableFlatFile::lookup_grib1(const char * parm_name, int table_number, int c n_matches = 0; // build a list of matches - vector matches; + vector matches; for(int j=0; j < N_grib1_elements; j++){ - if( g1e[j]->parm_name != parm_name || - (bad_data_int != table_number && g1e[j]->table_number != table_number) || - (bad_data_int != code && g1e[j]->code != code ) ) + if( g1e[j].parm_name != parm_name || + (bad_data_int != table_number && g1e[j].table_number != table_number) || + (bad_data_int != code && g1e[j].code != code) ) continue; - if( n_matches++ == 0 ) e = *(g1e[j]); + if( n_matches++ == 0 ) e = g1e[j]; matches.push_back( g1e[j] ); } @@ -1162,11 +1081,11 @@ bool TableFlatFile::lookup_grib1(const char * parm_name, int table_number, int c msg << "):\n"; mlog << Debug(3) << "\n" << msg; - for(vector::iterator it = matches.begin(); + for(vector::iterator it = matches.begin(); it < matches.end(); it++) - mlog << Debug(3) << " parm_name: " << (*it)->parm_name - << ", table_number = " << (*it)->table_number - << ", code = " << (*it)->code << "\n"; + mlog << Debug(3) << " parm_name: " << (it)->parm_name + << ", table_number = " << (it)->table_number + << ", code = " << (it)->code << "\n"; mlog << Debug(3) << "Using the first match found: " << " parm_name: " << e.parm_name @@ -1193,22 +1112,22 @@ bool TableFlatFile::lookup_grib1(const char * parm_name, int table_number, int c n_matches = 0; // build a list of matches - vector matches; + vector matches; int matching_subsenter; for(int j=0; j < N_grib1_elements; j++){ matching_subsenter = subcenter; - if( g1e[j]->subcenter == -1){ + if( g1e[j].subcenter == -1){ matching_subsenter = -1; } - if( g1e[j]->parm_name != parm_name || - (bad_data_int != table_number && g1e[j]->table_number != table_number) || - (bad_data_int != code && g1e[j]->code != code ) || - (bad_data_int != center && g1e[j]->center != center ) || - (bad_data_int != matching_subsenter && g1e[j]->subcenter != matching_subsenter) ) + if( g1e[j].parm_name != parm_name || + (bad_data_int != table_number && g1e[j].table_number != table_number) || + (bad_data_int != code && g1e[j].code != code) || + (bad_data_int != center && g1e[j].center != center) || + (bad_data_int != matching_subsenter && g1e[j].subcenter != matching_subsenter) ) continue; - if( n_matches++ == 0 ) e = *(g1e[j]); + if( n_matches++ == 0 ) e = g1e[j]; matches.push_back( g1e[j] ); } @@ -1224,14 +1143,14 @@ bool TableFlatFile::lookup_grib1(const char * parm_name, int table_number, int c msg << "):\n"; mlog << Debug(3) << "\n" << msg; - for(vector::iterator it = matches.begin(); + for(vector::iterator it = matches.begin(); it < matches.end(); it++) { - mlog << Debug(3) << " parm_name: " << (*it)->parm_name - << ", table_number = " << (*it)->table_number - << ", code = " << (*it)->code - << ", center = " << (*it)->center - << ", subcenter = " << (*it)->subcenter << "\n"; + mlog << Debug(3) << " parm_name: " << (it)->parm_name + << ", table_number = " << (it)->table_number + << ", code = " << (it)->code + << ", center = " << (it)->center + << ", subcenter = " << (it)->subcenter << "\n"; } mlog << Debug(3) << "Using the first match found: " @@ -1272,9 +1191,9 @@ e.clear(); for (j=0; jindex_a == a) && (g2e[j]->index_b == b) && (g2e[j]->index_c == c) ) { + if ( (g2e[j].index_a == a) && (g2e[j].index_b == b) && (g2e[j].index_c == c) ) { - e = *(g2e[j]); + e = g2e[j]; return ( true ); @@ -1302,17 +1221,17 @@ bool TableFlatFile::lookup_grib2(int a, int b, int c, for (j=0; jindex_a != a || - g2e[j]->index_b != b || - g2e[j]->index_c != c ) continue; + if ( g2e[j].index_a != a || + g2e[j].index_b != b || + g2e[j].index_c != c ) continue; // Check master table, center, and local table - if ( (bad_data_int != mtab && g2e[j]->mtab_low > mtab) || - (bad_data_int != mtab && g2e[j]->mtab_high < mtab) || - (bad_data_int != cntr && g2e[j]->cntr > 0 && g2e[j]->cntr != cntr) || - (bad_data_int != ltab && g2e[j]->ltab > 0 && g2e[j]->ltab != ltab) ) continue; + if ( (bad_data_int != mtab && g2e[j].mtab_low > mtab) || + (bad_data_int != mtab && g2e[j].mtab_high < mtab) || + (bad_data_int != cntr && g2e[j].cntr > 0 && g2e[j].cntr != cntr) || + (bad_data_int != ltab && g2e[j].ltab > 0 && g2e[j].ltab != ltab) ) continue; - e = *(g2e[j]); + e = g2e[j]; return ( true ); @@ -1335,16 +1254,16 @@ bool TableFlatFile::lookup_grib2(const char * parm_name, int a, int b, int c, n_matches = 0; // build a list of matches - vector matches; + vector matches; for(int j=0; jparm_name != parm_name || - (bad_data_int != a && g2e[j]->index_a != a) || - (bad_data_int != b && g2e[j]->index_b != b) || - (bad_data_int != c && g2e[j]->index_c != c) ) + if( g2e[j].parm_name != parm_name || + (bad_data_int != a && g2e[j].index_a != a) || + (bad_data_int != b && g2e[j].index_b != b) || + (bad_data_int != c && g2e[j].index_c != c) ) continue; - if( n_matches++ == 0 ) e = *(g2e[j]); + if( n_matches++ == 0 ) e = g2e[j]; matches.push_back( g2e[j] ); } @@ -1361,12 +1280,12 @@ bool TableFlatFile::lookup_grib2(const char * parm_name, int a, int b, int c, msg << "):\n"; mlog << Debug(3) << "\n" << msg; - for(vector::iterator it = matches.begin(); + for(vector::iterator it = matches.begin(); it < matches.end(); it++) - mlog << Debug(3) << " parm_name: " << (*it)->parm_name - << ", index_a = " << (*it)->index_a - << ", index_b = " << (*it)->index_b - << ", index_c = " << (*it)->index_c << "\n"; + mlog << Debug(3) << " parm_name: " << (it)->parm_name + << ", index_a = " << (it)->index_a + << ", index_b = " << (it)->index_b + << ", index_c = " << (it)->index_c << "\n"; mlog << Debug(3) << "Using the first match found: " << " parm_name: " << e.parm_name @@ -1395,20 +1314,20 @@ bool TableFlatFile::lookup_grib2(const char * parm_name, n_matches = 0; // build a list of matches - vector matches; + vector matches; for(int j=0; jparm_name != parm_name || - (bad_data_int != a && g2e[j]->index_a != a) || - (bad_data_int != b && g2e[j]->index_b != b) || - (bad_data_int != c && g2e[j]->index_c != c) || - (bad_data_int != mtab && g2e[j]->mtab_low > mtab) || - (bad_data_int != mtab && g2e[j]->mtab_high < mtab) || - (bad_data_int != cntr && g2e[j]->cntr > 0 && g2e[j]->cntr != cntr) || - (bad_data_int != ltab && g2e[j]->ltab > 0 && g2e[j]->ltab != ltab) ) + if( g2e[j].parm_name != parm_name || + (bad_data_int != a && g2e[j].index_a != a) || + (bad_data_int != b && g2e[j].index_b != b) || + (bad_data_int != c && g2e[j].index_c != c) || + (bad_data_int != mtab && g2e[j].mtab_low > mtab) || + (bad_data_int != mtab && g2e[j].mtab_high < mtab) || + (bad_data_int != cntr && g2e[j].cntr > 0 && g2e[j].cntr != cntr) || + (bad_data_int != ltab && g2e[j].ltab > 0 && g2e[j].ltab != ltab) ) continue; - if( n_matches++ == 0 ) e = *(g2e[j]); + if( n_matches++ == 0 ) e = g2e[j]; matches.push_back( g2e[j] ); } @@ -1428,15 +1347,15 @@ bool TableFlatFile::lookup_grib2(const char * parm_name, msg << "):\n"; mlog << Debug(3) << "\n" << msg; - for(vector::iterator it = matches.begin(); + for(vector::iterator it = matches.begin(); it < matches.end(); it++) - mlog << Debug(3) << " parm_name: " << (*it)->parm_name - << ", index_a = " << (*it)->index_a - << ", grib2_mtab = " << (*it)->mtab_set - << ", grib2_cntr = " << (*it)->cntr - << ", grib2_ltab = " << (*it)->ltab - << ", index_b = " << (*it)->index_b - << ", index_c = " << (*it)->index_c + mlog << Debug(3) << " parm_name: " << (it)->parm_name + << ", index_a = " << (it)->index_a + << ", grib2_mtab = " << (it)->mtab_set + << ", grib2_cntr = " << (it)->cntr + << ", grib2_ltab = " << (it)->ltab + << ", index_b = " << (it)->index_b + << ", index_c = " << (it)->index_c << "\n"; mlog << Debug(3) << "Using the first match found: " diff --git a/src/libcode/vx_data2d/table_lookup.h b/src/libcode/vx_data2d/table_lookup.h index a234bf72bb..e376293a4e 100644 --- a/src/libcode/vx_data2d/table_lookup.h +++ b/src/libcode/vx_data2d/table_lookup.h @@ -145,8 +145,8 @@ class TableFlatFile { void extend_grib1(int); void extend_grib2(int); - Grib1TableEntry ** g1e; // elements ... allocated - Grib2TableEntry ** g2e; // elements ... allocated + std::vector g1e; + std::vector g2e; int N_grib1_elements; int N_grib2_elements; @@ -229,5 +229,3 @@ extern TableFlatFile GribTable; //////////////////////////////////////////////////////////////////////// - - diff --git a/src/libcode/vx_data2d_python/python_dataplane.cc b/src/libcode/vx_data2d_python/python_dataplane.cc index f588db9c03..af886d7efe 100644 --- a/src/libcode/vx_data2d_python/python_dataplane.cc +++ b/src/libcode/vx_data2d_python/python_dataplane.cc @@ -23,7 +23,7 @@ //////////////////////////////////////////////////////////////////////// -GlobalPython GP; // this needs external linkage +extern GlobalPython GP; // this needs external linkage //////////////////////////////////////////////////////////////////////// @@ -35,12 +35,8 @@ static const char write_tmp_nc [] = "MET_BASE/wrappers/write_tmp_datapla static const char read_tmp_nc [] = "read_tmp_dataplane"; // NO ".py" suffix -static const char tmp_nc_base_name [] = "tmp_met_nc"; - static const char tmp_nc_var_name [] = "met_info"; -static const char tmp_nc_file_var_name [] = "tmp_nc_filename"; - //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_grid/grid_base.cc b/src/libcode/vx_grid/grid_base.cc index b909547805..88b2f85383 100644 --- a/src/libcode/vx_grid/grid_base.cc +++ b/src/libcode/vx_grid/grid_base.cc @@ -239,6 +239,23 @@ mlog << Debug(grid_debug_level) } +//////////////////////////////////////////////////////////////////////// + + +void SemiLatLonData::clear() + +{ + +name = (const char *) 0; + +lats.clear(); +lons.clear(); +levels.clear(); +times.clear(); + +} + + //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_grid/semilatlon_grid.cc b/src/libcode/vx_grid/semilatlon_grid.cc index 9018bc400e..76096ffb32 100644 --- a/src/libcode/vx_grid/semilatlon_grid.cc +++ b/src/libcode/vx_grid/semilatlon_grid.cc @@ -173,7 +173,7 @@ IsLatLon = false; Nx = 0; Ny = 0; -memset(&Data, 0, sizeof(Data)); +Data.clear(); return; diff --git a/src/libcode/vx_grid/semilatlon_grid_defs.h b/src/libcode/vx_grid/semilatlon_grid_defs.h index 5511007469..25529c66f0 100644 --- a/src/libcode/vx_grid/semilatlon_grid_defs.h +++ b/src/libcode/vx_grid/semilatlon_grid_defs.h @@ -29,6 +29,7 @@ struct SemiLatLonData { NumArray times; void dump(); + void clear(); }; diff --git a/src/libcode/vx_pointdata_python/python_pointdata.cc b/src/libcode/vx_pointdata_python/python_pointdata.cc index e99cc49596..e8bd3731d2 100644 --- a/src/libcode/vx_pointdata_python/python_pointdata.cc +++ b/src/libcode/vx_pointdata_python/python_pointdata.cc @@ -13,15 +13,30 @@ #include "python_pointdata.h" #include "pointdata_from_array.h" #include "vx_util.h" +#include "vx_python3_utils.h" #include "global_python.h" #include "wchar_argv.h" //////////////////////////////////////////////////////////////////////// +extern GlobalPython GP; // this needs external linkage + //////////////////////////////////////////////////////////////////////// +static const char * user_ppath = 0; + +static const char write_tmp_nc [] = "MET_BASE/wrappers/write_tmp_point_nc.py"; + +static const char read_tmp_nc [] = "read_tmp_point_nc"; // NO ".py" suffix + +//////////////////////////////////////////////////////////////////////// + + +static bool tmp_nc_point_obs(const char * script_name, int user_script_argc, + char ** user_script_argv, MetPointDataPython &met_pd_out); + static bool straight_python_point_data(const char * script_name, int script_argc, char ** script_argv, const bool use_xarray, MetPointDataPython &met_pd_out); @@ -52,13 +67,172 @@ bool python_point_data(const char * script_name, int script_argc, char ** script { -bool status = straight_python_point_data(script_name, script_argc, script_argv, - use_xarray, met_pd_out); +bool status = false; + +if ( user_ppath == 0 ) user_ppath = getenv(user_python_path_env); + +if ( user_ppath != 0 ) { // do_tmp_nc = true; + + status = tmp_nc_point_obs(script_name, script_argc, script_argv, + met_pd_out); +} +else { + status = straight_python_point_data(script_name, script_argc, script_argv, + use_xarray, met_pd_out); +} return ( status ); } +//////////////////////////////////////////////////////////////////////// + +bool process_python_point_data(PyObject *module_obj, MetPointDataPython &met_pd_out) +{ + +int int_value; +PyObject *module_dict_obj = 0; +PyObject *python_value = 0; +PyObject *python_met_point_data = 0; +ConcatString cs, user_dir, user_base; +const char *method_name = "process_python_point_data -> "; +const char *method_name_s = "process_python_point_data()"; + + // + // get the namespace for the module (as a dictionary) + // + +module_dict_obj = PyModule_GetDict (module_obj); + + // + // get handles to the objects of interest from the module_dict + // + +python_met_point_data = PyDict_GetItemString (module_dict_obj, python_key_point_data); + +python_value = PyDict_GetItemString (python_met_point_data, python_use_var_id); + +bool use_var_id = pyobject_as_bool(python_value); +met_pd_out.set_use_var_id(use_var_id); +mlog << Debug(9) << method_name << "use_var_id: \"" << use_var_id + << "\" from python. is_using_var_id(): " << met_pd_out.is_using_var_id() << "\n"; + +python_value = PyDict_GetItemString (python_met_point_data, python_key_nhdr); + +int_value = pyobject_as_int(python_value); +if (int_value == 0) { + mlog << Error << "\n" << method_name + << "The header is empty. Please check if python input exists\n\n"; + exit (1); +} +met_pd_out.set_hdr_cnt(int_value); + +python_value = PyDict_GetItemString (python_met_point_data, python_key_nobs); +int_value = pyobject_as_int(python_value); +if (int_value == 0) { + mlog << Error << "\n" << method_name + << "The point observation data is empty. Please check if python input is processed properly\n\n"; + exit (1); +} + +met_pd_out.allocate(int_value); + +MetPointObsData *obs_data = met_pd_out.get_point_obs_data(); +MetPointHeader *header_data = met_pd_out.get_header_data(); + + + // look up the data array variable name from the dictionary + + set_array_from_python(python_met_point_data, numpy_array_hdr_typ, &header_data->typ_idx_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_sid, &header_data->sid_idx_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_vld, &header_data->vld_idx_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_lat, &header_data->lat_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_lon, &header_data->lon_array); + set_array_from_python(python_met_point_data, numpy_array_hdr_elv, &header_data->elv_array); + if (header_data->typ_idx_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_typ is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->sid_idx_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_sid is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->vld_idx_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_vld is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->lat_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_lat is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->lon_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_lon is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->elv_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_elv is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + + set_str_array_from_python(python_met_point_data, numpy_array_hdr_typ_table, &header_data->typ_array); + set_str_array_from_python(python_met_point_data, numpy_array_hdr_sid_table, &header_data->sid_array); + set_str_array_from_python(python_met_point_data, numpy_array_hdr_vld_table, &header_data->vld_array); + if (header_data->typ_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_typ_table is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->sid_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_sid_table is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (header_data->vld_array.n() == 0) { + mlog << Error << "\n" << method_name + << "The hdr_vld_table is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + set_array_from_python(python_met_point_data, numpy_array_prpt_typ_table, &header_data->prpt_typ_array, false); + set_array_from_python(python_met_point_data, numpy_array_irpt_typ_table, &header_data->irpt_typ_array, false); + set_array_from_python(python_met_point_data, numpy_array_inst_typ_table, &header_data->inst_typ_array, false); + + set_array_from_python(python_met_point_data, numpy_array_obs_qty, obs_data->obs_qids); + set_array_from_python(python_met_point_data, numpy_array_obs_hid, obs_data->obs_hids); + set_array_from_python(python_met_point_data, numpy_array_obs_vid, obs_data->obs_ids); + set_array_from_python(python_met_point_data, numpy_array_obs_lvl, obs_data->obs_lvls); + set_array_from_python(python_met_point_data, numpy_array_obs_hgt, obs_data->obs_hgts); + set_array_from_python(python_met_point_data, numpy_array_obs_val, obs_data->obs_vals); + + set_str_array_from_python(python_met_point_data, numpy_array_obs_qty_table, &obs_data->qty_names); + set_str_array_from_python(python_met_point_data, numpy_array_obs_var_table, &obs_data->var_names); + if (obs_data->qty_names.n() == 0) { + mlog << Error << "\n" << method_name + << "The obs_qty_table is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + if (use_var_id && obs_data->var_names.n() == 0) { + mlog << Error << "\n" << method_name + << "The obs_var_table is empty. Please check if python input is processed properly\n\n"; + exit (1); + } + + if(mlog.verbosity_level()>=point_data_debug_level) print_met_data(obs_data, header_data, method_name_s); + + // + // done + // + +return ( true ); + +} + + //////////////////////////////////////////////////////////////////////// @@ -170,153 +344,175 @@ if ( ! module_obj ) { } + +return process_python_point_data(module_obj, met_pd_out); + +} + + +//////////////////////////////////////////////////////////////////////// + + +bool tmp_nc_point_obs(const char * user_script_name, int user_script_argc, + char ** user_script_argv, MetPointDataPython &met_pd_out) + +{ + +int j; +int status; +ConcatString command; +ConcatString path; +ConcatString tmp_nc_path; +const char * tmp_dir = 0; +Wchar_Argv wa; +const char *method_name = "tmp_nc_point_obs() -> "; + // - // get the namespace for the module (as a dictionary) + // if the global python object has already been initialized, + // we need to reload the module // -module_dict_obj = PyModule_GetDict (module_obj); +bool do_reload = GP.is_initialized; + +GP.initialize(); // - // get handles to the objects of interest from the module_dict + // start up the python interpreter // -python_met_point_data = PyDict_GetItemString (module_dict_obj, python_key_point_data); +if ( PyErr_Occurred() ) { -python_value = PyDict_GetItemString (python_met_point_data, python_use_var_id); + PyErr_Print(); -bool use_var_id = pyobject_as_bool(python_value); -met_pd_out.set_use_var_id(use_var_id); -mlog << Debug(9) << method_name << "use_var_id: \"" << use_var_id << "\" from python. is_using_var_id(): " << met_pd_out.is_using_var_id() << "\n"; + mlog << Warning << "\n" << method_name + << "an error occurred initializing python\n\n"; -python_value = PyDict_GetItemString (python_met_point_data, python_key_nhdr); + return ( false ); -int_value = pyobject_as_int(python_value); -if (int_value == 0) { - mlog << Error << "\n" << method_name - << "The header is empty. Please check if python input exists\n\n"; - exit (1); } -met_pd_out.set_hdr_cnt(int_value); -python_value = PyDict_GetItemString (python_met_point_data, python_key_nobs); -int_value = pyobject_as_int(python_value); -if (int_value == 0) { +run_python_string("import sys"); +command << cs_erase + << "sys.path.append(\"" + << replace_path(python_dir) + << "\")"; +run_python_string(command.text()); + +mlog << Debug(3) << "Calling " << user_ppath + << " to run user's python script (" << user_script_name + << ").\n"; + +tmp_dir = getenv ("MET_TMP_DIR"); + +if ( ! tmp_dir ) tmp_dir = default_tmp_dir; + +path << cs_erase + << tmp_dir << '/' + << tmp_nc_base_name; + +tmp_nc_path = make_temp_file_name(path.text(), 0); + +command << cs_erase + << user_ppath << ' ' // user's path to python + << replace_path(write_tmp_nc) << ' ' // write_tmp_nc.py + << tmp_nc_path << ' ' // tmp_nc output filename + << user_script_name; // user's script name + +for (j=1; jtyp_idx_array); - set_array_from_python(python_met_point_data, numpy_array_hdr_sid, &header_data->sid_idx_array); - set_array_from_python(python_met_point_data, numpy_array_hdr_vld, &header_data->vld_idx_array); - set_array_from_python(python_met_point_data, numpy_array_hdr_lat, &header_data->lat_array); - set_array_from_python(python_met_point_data, numpy_array_hdr_lon, &header_data->lon_array); - set_array_from_python(python_met_point_data, numpy_array_hdr_elv, &header_data->elv_array); - if (header_data->typ_idx_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_typ is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->sid_idx_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_sid is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->vld_idx_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_vld is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->lat_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_lat is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->lon_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_lon is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->elv_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_elv is empty. Please check if python input is processed properly\n\n"; - exit (1); - } + module_obj = PyImport_ReloadModule (module_obj); - set_str_array_from_python(python_met_point_data, numpy_array_hdr_typ_table, &header_data->typ_array); - set_str_array_from_python(python_met_point_data, numpy_array_hdr_sid_table, &header_data->sid_array); - set_str_array_from_python(python_met_point_data, numpy_array_hdr_vld_table, &header_data->vld_array); - if (header_data->typ_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_typ_table is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->sid_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_sid_table is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (header_data->vld_array.n() == 0) { - mlog << Error << "\n" << method_name - << "The hdr_vld_table is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - set_array_from_python(python_met_point_data, numpy_array_prpt_typ_table, &header_data->prpt_typ_array, false); - set_array_from_python(python_met_point_data, numpy_array_irpt_typ_table, &header_data->irpt_typ_array, false); - set_array_from_python(python_met_point_data, numpy_array_inst_typ_table, &header_data->inst_typ_array, false); +} - set_array_from_python(python_met_point_data, numpy_array_obs_qty, obs_data->obs_qids); - set_array_from_python(python_met_point_data, numpy_array_obs_hid, obs_data->obs_hids); - set_array_from_python(python_met_point_data, numpy_array_obs_vid, obs_data->obs_ids); - set_array_from_python(python_met_point_data, numpy_array_obs_lvl, obs_data->obs_lvls); - set_array_from_python(python_met_point_data, numpy_array_obs_hgt, obs_data->obs_hgts); - set_array_from_python(python_met_point_data, numpy_array_obs_val, obs_data->obs_vals); +if ( PyErr_Occurred() ) { - set_str_array_from_python(python_met_point_data, numpy_array_obs_qty_table, &obs_data->qty_names); - set_str_array_from_python(python_met_point_data, numpy_array_obs_var_table, &obs_data->var_names); - if (obs_data->qty_names.n() == 0) { - mlog << Error << "\n" << method_name - << "The obs_qty_table is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - if (use_var_id && obs_data->var_names.n() == 0) { - mlog << Error << "\n" << method_name - << "The obs_var_table is empty. Please check if python input is processed properly\n\n"; - exit (1); - } - - if(mlog.verbosity_level()>=point_data_debug_level) print_met_data(obs_data, header_data, method_name_s); + PyErr_Print(); + + mlog << Warning << "\n" << method_name + << "an error occurred importing module " + << '\"' << path << "\"\n\n"; + + return ( false ); } +if ( ! module_obj ) { + + mlog << Warning << "\n" << method_name + << "error running python script\n\n"; + + return ( false ); + +} + + // + // read the tmp_nc file + // + + // + // get the namespace for the module (as a dictionary) + // + + +process_python_point_data(module_obj, met_pd_out); + + + // + // cleanup + // + +remove_temp_file(tmp_nc_path); + // // done // diff --git a/src/libcode/vx_python3_utils/python3_util.cc b/src/libcode/vx_python3_utils/python3_util.cc index ab17acd7b8..f454c837b1 100644 --- a/src/libcode/vx_python3_utils/python3_util.cc +++ b/src/libcode/vx_python3_utils/python3_util.cc @@ -15,8 +15,13 @@ using namespace std; #include "vx_math.h" #include "python3_util.h" +#include "global_python.h" +//////////////////////////////////////////////////////////////////////// + +GlobalPython GP; // this needs external linkage + //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_python3_utils/python3_util.h b/src/libcode/vx_python3_utils/python3_util.h index b1681463c0..e81dfe260d 100644 --- a/src/libcode/vx_python3_utils/python3_util.h +++ b/src/libcode/vx_python3_utils/python3_util.h @@ -30,6 +30,13 @@ static const char user_python_path_env [] = "MET_PYTHON_EXE"; static const char wrappers_dir [] = "MET_BASE/wrappers"; +static const char python_dir [] = "MET_BASE/python"; + +static const char tmp_nc_base_name [] = "tmp_met_nc"; + +static const char tmp_nc_file_var_name [] = "tmp_nc_filename"; + +static const char tmp_nc_point_var_name[] = "met_point_data"; //////////////////////////////////////////////////////////////////////// @@ -56,7 +63,7 @@ extern PyObject * get_attribute(PyObject *, const char * attribute_name); extern int pyobject_as_int (PyObject *); -extern bool pyobject_as_bool (PyObject *); +extern bool pyobject_as_bool (PyObject *); extern double pyobject_as_double (PyObject *); extern std::string pyobject_as_string (PyObject *); extern ConcatString pyobject_as_concat_string (PyObject *); diff --git a/src/libcode/vx_statistics/contable.cc b/src/libcode/vx_statistics/contable.cc index 286f66d7ca..940c60598b 100644 --- a/src/libcode/vx_statistics/contable.cc +++ b/src/libcode/vx_statistics/contable.cc @@ -20,6 +20,7 @@ using namespace std; #include #include #include +#include #include "contable.h" @@ -255,15 +256,13 @@ out << prefix << "\n"; int n, m, k; int w, h; int r_table, c_table; -int * col_width = (int *) 0; -char * table = (char *) 0; const int hpad = 2; const int vpad = 1; const char v_sep = '|'; const char h_sep = '-'; const char corner_sep = '+'; -col_width = new int[Ncols]; +std::vector col_width(Ncols); for (c=0; c " - << "memory allocation error\n\n"; - - exit ( 1 ); - -} - -memset(table, ' ', w*h); +std::vector table(w*h, ' '); // // top, bottom @@ -453,10 +441,6 @@ for (r_table=0; r_table @@ -26,15 +24,12 @@ using namespace std; //////////////////////////////////////////////////////////////////////// - // // Code for class AirnowLocations // - //////////////////////////////////////////////////////////////////////// - AirnowLocations::AirnowLocations() { // @@ -43,27 +38,24 @@ AirnowLocations::AirnowLocations() monitoringSiteFileName = "Monitoring_Site_Locations_V2.dat"; } - //////////////////////////////////////////////////////////////////////// - AirnowLocations::~AirnowLocations() { } - //////////////////////////////////////////////////////////////////////// bool AirnowLocations::initialize(const string &fileName) { - string method_name = "AirnowLocations::initialize() "; + string method_name = "AirnowLocations::initialize()"; monitoringSiteFileName = fileName; LineDataFile locFile; if (!locFile.open(monitoringSiteFileName.c_str())) { - mlog << Error << method_name << "->" - << "can't open input ASCII file \"" << monitoringSiteFileName - << "\" for reading\n\n"; + mlog << Warning << "\n" << method_name << " -> " + << "can't open input ASCII file \"" << monitoringSiteFileName + << "\" for reading\n\n"; return false; } DataLine data_line; @@ -119,12 +111,12 @@ bool AirnowLocations::initialize(const string &fileName) double lon2 = monitoringSiteLon[index]; double elev2 = monitoringSiteElev[index]; if (lat != lat2 || lon != lon2 || elev != elev2 || aqsid2 != aqsid || stationid2 != stationid) { - mlog << Warning << "AirnowLocations" << method_name << "-> " - << "Multiple values seen for a single FullAQSID (" << fullaqsid << ")" - << "Values used: StationId:" << stationid << " Aqsid:" << aqsid << " Lat,Lon,Elev:" - << lat << "," << lon << "," << elev - << "Not used StationId:" << stationid2 << " Aqsid:" << aqsid2 << " Lat,Lon,Elev:" - << lat2 << "," << lon2 << "," << elev2 << "\n\n"; + mlog << Warning << "\n" << method_name << " -> " + << "Multiple values seen for a single FullAQSID (" << fullaqsid << ")" + << "Values used: StationId:" << stationid << " Aqsid:" << aqsid << " Lat,Lon,Elev:" + << lat << "," << lon << "," << elev + << "Not used StationId:" << stationid2 << " Aqsid:" << aqsid2 << " Lat,Lon,Elev:" + << lat2 << "," << lon2 << "," << elev2 << "\n\n"; } } } @@ -157,12 +149,12 @@ bool AirnowLocations::lookupLatLonElev(const string aqsid, double &lat, double & if (it == monitoringSiteStationId.end()) { it = find(monitoringSiteFullAqsid.begin(), monitoringSiteFullAqsid.end(), aqsid); if (it == monitoringSiteFullAqsid.end()) { - return false; + return false; } else { - index = (int)(it - monitoringSiteStationId.begin()); + index = (int)(it - monitoringSiteFullAqsid.begin()); } } else { - index = (int)(it - monitoringSiteFullAqsid.begin()); + index = (int)(it - monitoringSiteStationId.begin()); } } else { index = (int)(it - monitoringSiteAqsid.begin()); @@ -188,9 +180,10 @@ bool AirnowLocations::_setPtr(DataLine &data_line, const string &headerName, int return true; } } - mlog << Error << "AirnowLocations::_setPtr() ->" + mlog << Warning << "\nAirnowLocations::_setPtr() -> " << "Did not see '" << headerName << "' in the header line of file " - << monitoringSiteFileName << "\n"; + << monitoringSiteFileName << "\n\n"; return false; } +//////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/mode_graphics/cgraph_font.cc b/src/tools/other/mode_graphics/cgraph_font.cc index 9f12dca012..335d081f6c 100644 --- a/src/tools/other/mode_graphics/cgraph_font.cc +++ b/src/tools/other/mode_graphics/cgraph_font.cc @@ -115,7 +115,7 @@ face = 0; if ( !get_env(cg_font_env, gs_font_dir) ) { mlog << Error - << "\n\n CgFont::init_from_scratch() -> " + << "\nCgFont::init_from_scratch() -> " << "unable to get environment variable \"" << cg_font_env << "\"\n\n"; @@ -258,7 +258,8 @@ clear(); if ( (n < 0) || (n >= total_predef_fonts) ) { - mlog << Error << "\n\n CgFont::set_by_number(int) -> range check error\n\n"; + mlog << Error << "\nCgFont::set_by_number(int) -> " + << "range check error\n\n"; exit ( 1 ); @@ -274,7 +275,8 @@ full_afm_name << gs_font_dir << '/' << gs_font_dir << '/' << short_afm_name; if ( !file_exists(full_afm_name.c_str()) ) { - mlog << Error << "\n\n CgFont::set_by_number(int) -> can't find afm file \"" << full_afm_name << "\"\n\n"; + mlog << Error << "\nCgFont::set_by_number(int) -> " + << "can't find afm file \"" << full_afm_name << "\"\n\n"; exit ( 1 ); @@ -284,7 +286,8 @@ afm = new Afm; if ( !(afm->read(full_afm_name)) ) { - mlog << Error << "\n\n CgFont::set_by_number(int) -> trouble reading afm file \"" << full_afm_name << "\"\n\n"; + mlog << Error << "\nCgFont::set_by_number(int) -> " + << "trouble reading afm file \"" << full_afm_name << "\"\n\n"; exit ( 1 ); @@ -374,7 +377,7 @@ void CgFontCollection::init_from_scratch() { -e = (CgFont **) 0; +Nelements = Nalloc = 0; clear(); @@ -390,34 +393,28 @@ void CgFontCollection::clear() { -if ( e ) { - - int j, error; - - for (j=0; jface); +int j, error; - if ( error ) { - - mlog << Error << "\n\n CgFontCollection::clear() -> trouble closing typeface \"" << e[j]->short_pfb_name << "\"\n\n"; +for (j=0; jface = 0; + mlog << Error << "\nCgFontCollection::clear() -> " + << "trouble closing typeface \"" << e[j].short_pfb_name + << "\"\n\n"; - if ( e[j] ) { delete e[j]; e[j] = (CgFont *) 0; } + exit ( 1 ); } - delete [] e; e = (CgFont **) 0; + e[j].face = 0; } +e.clear(); + Nelements = Nalloc = 0; return; @@ -445,7 +442,7 @@ for (j=0; jdump(out, depth + 1); + e[j].dump(out, depth + 1); } @@ -470,37 +467,13 @@ void CgFontCollection::extend(int n) if ( n <= Nalloc ) return; -int j, k; -CgFont ** u = (CgFont **) 0; - - -k = n/alloc_inc; +int k = n/alloc_inc; if ( n%alloc_inc ) ++k; n = k*alloc_inc; -u = new CgFont * [n]; - -if ( !u ) { - - mlog << Error << "\n\n CgFontCollection::extend(int) -> memory allocation error\n\n"; - - exit ( 1 ); - -} - -for (j=0; jps_font_number == n ) return ( e[j] ); + if ( e[j].ps_font_number == n ) return ( & e[j] ); } @@ -641,7 +596,7 @@ return ( (CgFont *) 0 ); //////////////////////////////////////////////////////////////////////// -CgFont * CgFontCollection::lookup_by_ps_name(const char * name) const +CgFont * CgFontCollection::lookup_by_ps_name(const char * name) { @@ -649,7 +604,7 @@ int j; for (j=0; jps_name == name ) return ( e[j] ); + if ( e[j].ps_name == name ) return ( & e[j] ); } @@ -662,19 +617,20 @@ return ( (CgFont *) 0 ); //////////////////////////////////////////////////////////////////////// -CgFont * CgFontCollection::operator[](int k) const +CgFont * CgFontCollection::operator[](int k) { if ( (k < 0) || (k >= Nelements) ) { - mlog << Error << "\n\n CgFont * CgFontCollection::operator[](int) const -> range check error\n\n"; + mlog << Error << "\nCgFont * CgFontCollection::operator[](int) const -> " + << "range check error\n\n"; exit ( 1 ); } -return ( e[k] ); +return ( & e[k] ); } @@ -702,8 +658,3 @@ return ( false ); //////////////////////////////////////////////////////////////////////// - - - - - diff --git a/src/tools/other/mode_graphics/cgraph_font.h b/src/tools/other/mode_graphics/cgraph_font.h index f8e4f8b493..287a2c289f 100644 --- a/src/tools/other/mode_graphics/cgraph_font.h +++ b/src/tools/other/mode_graphics/cgraph_font.h @@ -18,6 +18,7 @@ #include +#include #include "ft2build.h" #include FT_FREETYPE_H @@ -98,7 +99,7 @@ class CgFontCollection { int Nalloc; - CgFont ** e; + std::vector e; public: @@ -117,12 +118,11 @@ class CgFontCollection { void add (const CgFont &); void add_no_repeat (const CgFont &); - CgFont * lookup_by_ps_name(const char *) const; + CgFont * lookup_by_ps_name(const char *); - CgFont * lookup_by_ps_font_number(int) const; // for builtin fonts + CgFont * lookup_by_ps_font_number(int); // for builtin fonts - - CgFont * operator[](int) const; + CgFont * operator[](int); };