From d26e62d13b9901af667132c6a41c3fc6a2a69d2a Mon Sep 17 00:00:00 2001 From: David Fillmore Date: Fri, 2 Apr 2021 12:21:00 -0600 Subject: [PATCH] Feature 1319 no pickle (#1720) * Try path insert. * sys.path insert. * Per #1319, adding David's changes back into the feature_1319_no_pickle branch. It compiles but TEST: python_numpy_plot_data_plane_pickle fails when testing on my Mac. Comitting now to test on kiowa. * Per #1319, small updated to write_tmp_dataplane.py script. Had a couple of if statements that should really be elif. Co-authored-by: John Halley Gotway Co-authored-by: John Halley Gotway --- met/data/wrappers/Makefile.am | 7 +- met/data/wrappers/read_pickle_dataplane.py | 15 --- met/data/wrappers/read_tmp_ascii.py | 49 +++++++ met/data/wrappers/read_tmp_dataplane.py | 36 ++++++ met/data/wrappers/write_pickle_dataplane.py | 42 ------ met/data/wrappers/write_tmp_dataplane.py | 68 ++++++++++ ...ite_pickle_point.py => write_tmp_point.py} | 13 +- .../vx_data2d_python/python_dataplane.cc | 54 ++++---- .../vx_python3_utils/python3_script.cc | 121 +++++++++++++++++- .../libcode/vx_python3_utils/python3_script.h | 17 ++- met/src/tools/other/ascii2nc/ascii2nc.cc | 1 + .../tools/other/ascii2nc/python_handler.cc | 48 +++---- met/src/tools/other/ascii2nc/python_handler.h | 10 +- 13 files changed, 355 insertions(+), 126 deletions(-) delete mode 100644 met/data/wrappers/read_pickle_dataplane.py create mode 100644 met/data/wrappers/read_tmp_ascii.py create mode 100644 met/data/wrappers/read_tmp_dataplane.py delete mode 100644 met/data/wrappers/write_pickle_dataplane.py create mode 100644 met/data/wrappers/write_tmp_dataplane.py rename met/data/wrappers/{write_pickle_point.py => write_tmp_point.py} (69%) diff --git a/met/data/wrappers/Makefile.am b/met/data/wrappers/Makefile.am index d8a6d5a026..f3518fa27b 100644 --- a/met/data/wrappers/Makefile.am +++ b/met/data/wrappers/Makefile.am @@ -23,10 +23,11 @@ wrappersdir = $(pkgdatadir)/wrappers wrappers_DATA = \ generic_python.py \ generic_pickle.py \ - read_pickle_dataplane.py \ - write_pickle_dataplane.py \ + read_tmp_dataplane.py \ + write_tmp_dataplane.py \ write_pickle_mpr.py \ - write_pickle_point.py + read_tmp_ascii.py \ + write_tmp_point.py EXTRA_DIST = ${wrappers_DATA} diff --git a/met/data/wrappers/read_pickle_dataplane.py b/met/data/wrappers/read_pickle_dataplane.py deleted file mode 100644 index f97f153df7..0000000000 --- a/met/data/wrappers/read_pickle_dataplane.py +++ /dev/null @@ -1,15 +0,0 @@ -######################################################################## -# -# Reads temporary pickle file into memory. -# -# usage: /path/to/python read_pickle_dataplane.py pickle.tmp -# -######################################################################## - -import sys -import numpy as np -import pickle - -print('Python Script:\t', sys.argv[0]) -print('Load Pickle:\t', sys.argv[1]) -met_info = pickle.load(open(sys.argv[1], "rb")) diff --git a/met/data/wrappers/read_tmp_ascii.py b/met/data/wrappers/read_tmp_ascii.py new file mode 100644 index 0000000000..b4f4303044 --- /dev/null +++ b/met/data/wrappers/read_tmp_ascii.py @@ -0,0 +1,49 @@ +""" +Module Name: read_tmp_ascii.py + +Read MET Point Observations from a text file created by write_tmp_point.py script + or MET Matched Pairs from a text file created by write_tmp_mpr.py script + +Point observation format: + Message_Type, Station_ID, Valid_Time, Lat, Lon, Elevation, + GRIB_Code or Variable_Name, Level, Height, QC_String, Observation_Value + +Version Date +1.0.0 2021/02/18 David Fillmore Initial version +""" + +__author__ = 'David Fillmore' +__version__ = '1.0.0' +__email__ = 'met_help@ucar.edu' + +import argparse + +point_data = None + +def read_tmp_ascii(filename): + """ + Arguments: + filename (string): temporary file created by write_tmp_point.py + + Returns: + (list of lists): point data + """ + print('read_tmp_ascii:' + filename) + f = open(filename, 'r') + lines = f.readlines() + f.close() + + global point_data + point_data = [eval(line.strip('\n')) for line in lines] + + return point_data + +if __name__ == '__main__': + """ + Parse command line arguments + """ + parser = argparse.ArgumentParser() + parser.add_argument('--filename', type=str) + args = parser.parse_args() + + data = read_tmp_ascii(args.filename) diff --git a/met/data/wrappers/read_tmp_dataplane.py b/met/data/wrappers/read_tmp_dataplane.py new file mode 100644 index 0000000000..e21c17ba3f --- /dev/null +++ b/met/data/wrappers/read_tmp_dataplane.py @@ -0,0 +1,36 @@ +######################################################################## +# +# Reads temporary file into memory. +# +# usage: /path/to/python read_tmp_dataplane.py dataplane.tmp +# +######################################################################## + +import sys +import numpy as np +import netCDF4 as nc + +print('Python Script:\t', sys.argv[0]) +met_info = {} + +netcdf_filename = sys.argv[1] +print('Read NetCDF:\t', netcdf_filename) + +# read NetCDF file +ds = nc.Dataset(netcdf_filename, 'r') +met_data = ds['met_data'][:] +met_attrs = {} +grid = {} +for attr, attr_val in ds.__dict__.items(): + if 'grid' in attr: + grid_attr = attr.split('.')[1] + grid[grid_attr] = attr_val + else: + met_attrs[attr] = attr_val +grid['nx'], grid['ny'] = int(grid['nx']), int(grid['ny']) +met_attrs['grid'] = grid +met_attrs['name'] = met_attrs['name_str'] +del met_attrs['name_str'] +met_info['met_data'] = met_data +met_info['attrs'] = met_attrs +print(met_info) diff --git a/met/data/wrappers/write_pickle_dataplane.py b/met/data/wrappers/write_pickle_dataplane.py deleted file mode 100644 index 079557538b..0000000000 --- a/met/data/wrappers/write_pickle_dataplane.py +++ /dev/null @@ -1,42 +0,0 @@ -######################################################################## -# -# Adapted from a script provided by George McCabe -# Adapted by Randy Bullock -# -# usage: /path/to/python write_pickle_dataplane.py \ -# pickle_output_filename .py -# -######################################################################## - -import os -import sys -import pickle -import importlib.util -import xarray as xr - -print('Python Script:\t', sys.argv[0]) -print('User Command:\t', sys.argv[2:]) -print('Write Pickle:\t', sys.argv[1]) - -pickle_filename = sys.argv[1] - -pyembed_module_name = sys.argv[2] -sys.argv = sys.argv[2:] - -if not pyembed_module_name.endswith('.py'): - pyembed_module_name += '.py' - -user_base = os.path.basename(pyembed_module_name).replace('.py','') - -spec = importlib.util.spec_from_file_location(user_base, pyembed_module_name) -met_in = importlib.util.module_from_spec(spec) -spec.loader.exec_module(met_in) - -if isinstance(met_in.met_data, xr.DataArray): - met_info = { 'attrs': met_in.met_data.attrs, 'met_data': met_in.met_data } -else: - met_info = { 'attrs': met_in.attrs, 'met_data': met_in.met_data } - -print(met_info) - -pickle.dump( met_info, open( pickle_filename, "wb" ) ) diff --git a/met/data/wrappers/write_tmp_dataplane.py b/met/data/wrappers/write_tmp_dataplane.py new file mode 100644 index 0000000000..b5a1b24995 --- /dev/null +++ b/met/data/wrappers/write_tmp_dataplane.py @@ -0,0 +1,68 @@ +######################################################################## +# +# Adapted from a script provided by George McCabe +# Adapted by Randy Bullock +# +# usage: /path/to/python write_tmp_dataplane.py \ +# tmp_output_filename .py +# +######################################################################## + +import os +import sys +import importlib.util +import netCDF4 as nc + +print('Python Script:\t', sys.argv[0]) +print('User Command:\t', sys.argv[2:]) + +netcdf_filename = sys.argv[1] + +print('Write NetCDF:\t', netcdf_filename) + +pyembed_module_name = sys.argv[2] +sys.argv = sys.argv[2:] + +# append user script dir to system path +pyembed_dir, pyembed_file = os.path.split(pyembed_module_name) +if pyembed_dir: + sys.path.insert(0, pyembed_dir) + +if not pyembed_module_name.endswith('.py'): + pyembed_module_name += '.py' + +user_base = os.path.basename(pyembed_module_name).replace('.py','') + +spec = importlib.util.spec_from_file_location(user_base, pyembed_module_name) +met_in = importlib.util.module_from_spec(spec) +spec.loader.exec_module(met_in) + +met_info = {'met_data': met_in.met_data} +if hasattr(met_in.met_data, 'attrs') and met_in.met_data.attrs: + attrs = met_in.met_data.attrs +else: + attrs = met_in.attrs +met_info['attrs'] = attrs + +print('write_tmp_dataplane') +print(met_info) + +# write NetCDF file +ds = nc.Dataset(netcdf_filename, 'w') + +nx, ny = met_in.met_data.shape +ds.createDimension('x', nx) +ds.createDimension('y', ny) +dp = ds.createVariable('met_data', met_in.met_data.dtype, ('x', 'y')) +dp[:] = met_in.met_data + +for attr, attr_val in met_info['attrs'].items(): + print(attr, attr_val, type(attr_val)) + if attr == 'name': + setattr(ds, 'name_str', attr_val) + elif type(attr_val) == str: + setattr(ds, attr, attr_val) + elif type(attr_val) == dict: + for key in attr_val: + setattr(ds, attr + '.' + key, attr_val[key]) +ds.close() diff --git a/met/data/wrappers/write_pickle_point.py b/met/data/wrappers/write_tmp_point.py similarity index 69% rename from met/data/wrappers/write_pickle_point.py rename to met/data/wrappers/write_tmp_point.py index 1f5ee35bdb..94f56cd3dd 100644 --- a/met/data/wrappers/write_pickle_point.py +++ b/met/data/wrappers/write_tmp_point.py @@ -3,21 +3,20 @@ # Adapted from a script provided by George McCabe # Adapted by Randy Bullock # -# usage: /path/to/python write_pickle_point.py \ -# pickle_output_filename .py +# usage: /path/to/python write_tmp_point.py \ +# tmp_ascii_output_filename .py # ######################################################################## import os import sys -import pickle import importlib.util print('Python Script:\t', sys.argv[0]) print('User Command:\t', sys.argv[2:]) -print('Write Pickle:\t', sys.argv[1]) +print('Write Temporary Ascii:\t', sys.argv[1]) -pickle_filename = sys.argv[1] +tmp_filename = sys.argv[1] pyembed_module_name = sys.argv[2] sys.argv = sys.argv[2:] @@ -28,4 +27,6 @@ met_in = importlib.util.module_from_spec(spec) spec.loader.exec_module(met_in) -pickle.dump( met_in.point_data, open( pickle_filename, "wb" ) ) +f = open(tmp_filename, 'w') +for line in met_in.point_data: + f.write(str(line) + '\n') diff --git a/met/src/libcode/vx_data2d_python/python_dataplane.cc b/met/src/libcode/vx_data2d_python/python_dataplane.cc index d5ace046d0..8f70af5109 100644 --- a/met/src/libcode/vx_data2d_python/python_dataplane.cc +++ b/met/src/libcode/vx_data2d_python/python_dataplane.cc @@ -31,15 +31,15 @@ GlobalPython GP; // this needs external linkage static const char * user_ppath = 0; -static const char write_pickle [] = "MET_BASE/wrappers/write_pickle_dataplane.py"; +static const char write_tmp_nc [] = "MET_BASE/wrappers/write_tmp_dataplane.py"; -static const char read_pickle [] = "read_pickle_dataplane"; // NO ".py" suffix +static const char read_tmp_nc [] = "read_tmp_dataplane"; // NO ".py" suffix -static const char pickle_base_name [] = "tmp_met_pickle"; +static const char tmp_nc_base_name [] = "tmp_met_nc"; -static const char pickle_var_name [] = "met_info"; +static const char tmp_nc_var_name [] = "met_info"; -static const char pickle_file_var_name [] = "pickle_filename"; +static const char tmp_nc_file_var_name [] = "tmp_nc_filename"; //////////////////////////////////////////////////////////////////////// @@ -51,7 +51,7 @@ static bool straight_python_dataplane(const char * script_name, Grid & met_grid_out, VarInfoPython &vinfo); -static bool pickle_dataplane(const char * script_name, +static bool tmp_nc_dataplane(const char * script_name, int script_argc, char ** script_argv, const bool use_xarray, DataPlane & met_dp_out, Grid & met_grid_out, VarInfoPython &vinfo); @@ -69,9 +69,9 @@ bool python_dataplane(const char * user_script_name, bool status = false; -if ( (user_ppath = getenv(user_python_path_env)) != 0 ) { // do_pickle = true; +if ( (user_ppath = getenv(user_python_path_env)) != 0 ) { // do_tmp_nc = true; - status = pickle_dataplane(user_script_name, + status = tmp_nc_dataplane(user_script_name, user_script_argc, user_script_argv, use_xarray, met_dp_out, met_grid_out, vinfo); @@ -276,7 +276,7 @@ return ( true ); //////////////////////////////////////////////////////////////////////// -bool pickle_dataplane(const char * user_script_name, +bool tmp_nc_dataplane(const char * user_script_name, int user_script_argc, char ** user_script_argv, const bool use_xarray, DataPlane & met_dp_out, Grid & met_grid_out, VarInfoPython &vinfo) @@ -287,7 +287,7 @@ int j; int status; ConcatString command; ConcatString path; -ConcatString pickle_path; +ConcatString tmp_nc_path; const char * tmp_dir = 0; Wchar_Argv wa; @@ -301,14 +301,14 @@ if ( ! tmp_dir ) tmp_dir = default_tmp_dir; path << cs_erase << tmp_dir << '/' - << pickle_base_name; + << tmp_nc_base_name; -pickle_path = make_temp_file_name(path.text(), 0); +tmp_nc_path = make_temp_file_name(path.text(), 0); command << cs_erase << user_ppath << ' ' // user's path to python - << replace_path(write_pickle) << ' ' // write_pickle.py - << pickle_path << ' ' // pickle output filename + << replace_path(write_tmp_nc) << ' ' // write_tmp_nc.py + << tmp_nc_path << ' ' // tmp_nc output filename << user_script_name; // user's script name for (j=1; j " + mlog << Error << "\ntmp_nc_dataplane() -> " << "command \"" << command.text() << "\" failed ... status = " << status << "\n\n"; @@ -346,15 +346,15 @@ if ( PyErr_Occurred() ) { PyErr_Print(); - mlog << Warning << "\npickle_dataplane() -> " + mlog << Warning << "\ntmp_nc_dataplane() -> " << "an error occurred initializing python\n\n"; return ( false ); } -mlog << Debug(3) << "Reading temporary pickle file: " - << pickle_path << "\n"; +mlog << Debug(3) << "Reading temporary tmp_nc file: " + << tmp_nc_path << "\n"; // // set the arguments @@ -362,9 +362,9 @@ mlog << Debug(3) << "Reading temporary pickle file: " StringArray a; -a.add(read_pickle); +a.add(read_tmp_nc); -a.add(pickle_path); +a.add(tmp_nc_path); wa.set(a); @@ -374,7 +374,7 @@ PySys_SetArgv (wa.wargc(), wa.wargv()); // import the python wrapper script as a module // -path = get_short_name(read_pickle); +path = get_short_name(read_tmp_nc); PyObject * module_obj = PyImport_ImportModule (path.text()); @@ -392,7 +392,7 @@ if ( PyErr_Occurred() ) { PyErr_Print(); - mlog << Warning << "\npickle_dataplane() -> " + mlog << Warning << "\ntmp_nc_dataplane() -> " << "an error occurred importing module " << '\"' << path << "\"\n\n"; @@ -402,7 +402,7 @@ if ( PyErr_Occurred() ) { if ( ! module_obj ) { - mlog << Warning << "\npickle_dataplane() -> " + mlog << Warning << "\ntmp_nc_dataplane() -> " << "error running python script\n\n"; return ( false ); @@ -410,7 +410,7 @@ if ( ! module_obj ) { } // - // read the pickle file + // read the tmp_nc file // // @@ -419,13 +419,13 @@ if ( ! module_obj ) { PyObject * module_dict_obj = PyModule_GetDict (module_obj); -PyObject * key_obj = PyUnicode_FromString (pickle_var_name); +PyObject * key_obj = PyUnicode_FromString (tmp_nc_var_name); PyObject * data_obj = PyDict_GetItem (module_dict_obj, key_obj); if ( ! data_obj || ! PyDict_Check(data_obj) ) { - mlog << Error << "\npickle_dataplane() -> " + mlog << Error << "\ntmp_nc_dataplane() -> " << "bad dict object\n\n"; exit ( 1 ); @@ -450,7 +450,7 @@ dataplane_from_numpy_array(np, attrs_dict_obj, met_dp_out, met_grid_out, vinfo); // cleanup // -remove_temp_file(pickle_path); +remove_temp_file(tmp_nc_path); // // done diff --git a/met/src/libcode/vx_python3_utils/python3_script.cc b/met/src/libcode/vx_python3_utils/python3_script.cc index 56837b65d0..7bd567ae2c 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.cc +++ b/met/src/libcode/vx_python3_utils/python3_script.cc @@ -27,6 +27,8 @@ using namespace std; static const char sq = '\''; // single quote +static const char read_tmp_ascii_py [] = "MET_BASE/wrappers/read_tmp_ascii.py"; + //////////////////////////////////////////////////////////////////////// @@ -75,8 +77,12 @@ void Python3_Script::clear() Module = 0; +ModuleAscii = 0; + Dict = 0; +DictAscii = 0; + Script_Filename.clear(); @@ -159,14 +165,29 @@ return ( var ); } +//////////////////////////////////////////////////////////////////////// + +PyObject * Python3_Script::lookup_ascii(const char * name) const + +{ + +PyObject * var = 0; + +var = PyDict_GetItemString (DictAscii, name); + +return ( var ); + +} //////////////////////////////////////////////////////////////////////// -void Python3_Script::run(const char * command) const +PyObject * Python3_Script::run(const char * command) const { +PyObject * pobj; + if ( empty(command) ) { mlog << Error << "\nPython3_Script::run(const char *) -> " @@ -176,7 +197,9 @@ if ( empty(command) ) { } -if ( ! PyRun_String(command, Py_file_input, Dict, Dict) ) { +pobj = PyRun_String(command, Py_file_input, Dict, Dict); + +if ( ! pobj ) { mlog << Error << "\nPython3_Script::run(const char *) -> " << "command \"" << command << "\" failed!\n\n"; @@ -188,7 +211,7 @@ if ( ! PyRun_String(command, Py_file_input, Dict, Dict) ) { fflush(stdout); fflush(stderr); -return; +return pobj; } @@ -234,6 +257,98 @@ return; } +//////////////////////////////////////////////////////////////////////// + +void Python3_Script::import_read_tmp_ascii_py(void) + +{ + +ConcatString module; + +module << cs_erase + << replace_path(read_tmp_ascii_py); + +ConcatString command; + +run_python_string("import sys"); + +command << cs_erase + << "sys.path.append(\"" + << module.dirname().c_str() + << "\")"; + +mlog << Debug(3) << command << "\n"; + +run_python_string(command.text()); + +mlog << Debug(2) << "Importing " << module << "\n"; + +ConcatString path = "read_tmp_ascii"; + +ModuleAscii = PyImport_ImportModule(path.text()); + +if ( ! ModuleAscii ) { + + PyErr_Print(); + mlog << Error << "\nPython3_Script::Python3_Script(const char *) -> " + << "unable to import module \"" << path << "\"\n\n"; + + Py_Finalize(); + + exit ( 1 ); + +} + +DictAscii = PyModule_GetDict(ModuleAscii); + + // + // done + // + +fflush(stdout); +fflush(stderr); + +} + +//////////////////////////////////////////////////////////////////////// + +PyObject* Python3_Script::read_tmp_ascii(const char * tmp_filename) const + +{ + +mlog << Debug(2) << "Reading temporary ascii file: " + << tmp_filename << "\n"; + +ConcatString command; + +command << "read_tmp_ascii(\"" + << tmp_filename + << "\")"; + +mlog << Debug(3) << command << "\n"; + +PyErr_Clear(); + +PyObject * pobj; + +pobj = PyRun_String(command.text(), Py_file_input, DictAscii, DictAscii); + +if ( PyErr_Occurred() ) { + + mlog << Error << "\nPython3_Script::read_tmp_ascii() -> " + << "command \"" << command << "\" failed!\n\n"; + + exit ( 1 ); +} + +PyTypeObject* type = pobj->ob_type; + +const char* p = type->tp_name; + +mlog << Debug(2) << "read_tmp_ascii return type: " << p << "\n"; + +return pobj; +} //////////////////////////////////////////////////////////////////////// diff --git a/met/src/libcode/vx_python3_utils/python3_script.h b/met/src/libcode/vx_python3_utils/python3_script.h index 20069762f9..6930d226a5 100644 --- a/met/src/libcode/vx_python3_utils/python3_script.h +++ b/met/src/libcode/vx_python3_utils/python3_script.h @@ -41,6 +41,10 @@ class Python3_Script { PyObject * Dict; // script dictionary, not allocated + PyObject * ModuleAscii; + + PyObject * DictAscii; + ConcatString Script_Filename; @@ -62,6 +66,8 @@ class Python3_Script { PyObject * module(); PyObject * dict(); + PyObject * module_ascii(); + PyObject * dict_ascii(); // // do stuff @@ -73,10 +79,15 @@ class Python3_Script { PyObject * lookup(const char * name) const; - void run(const char * command) const; // runs a command in the namespace of the script + PyObject * lookup_ascii(const char * name) const; + + PyObject * run(const char * command) const; // runs a command in the namespace of the script void read_pickle (const char * variable_name, const char * pickle_filename) const; + void import_read_tmp_ascii_py (void); + + PyObject * read_tmp_ascii (const char * tmp_filename) const; }; @@ -87,6 +98,10 @@ inline PyObject * Python3_Script::module() { return ( Module ); } inline PyObject * Python3_Script::dict() { return ( Dict ); } +inline PyObject * Python3_Script::module_ascii() { return ( ModuleAscii ); } + +inline PyObject * Python3_Script::dict_ascii() { return ( DictAscii ); } + inline ConcatString Python3_Script::filename() const { return ( Script_Filename ); } diff --git a/met/src/tools/other/ascii2nc/ascii2nc.cc b/met/src/tools/other/ascii2nc/ascii2nc.cc index 4ced5397b1..360329659c 100644 --- a/met/src/tools/other/ascii2nc/ascii2nc.cc +++ b/met/src/tools/other/ascii2nc/ascii2nc.cc @@ -43,6 +43,7 @@ // 015 03-20-19 Fillmore Add aeronetv2 and aeronetv3 options. // 016 01-30-20 Bullock Add python option. // 017 01-25-21 Halley Gotway MET #1630 Handle zero obs. +// 018 03-01-21 Fillmore Replace pickle files for temporary ascii. // //////////////////////////////////////////////////////////////////////// diff --git a/met/src/tools/other/ascii2nc/python_handler.cc b/met/src/tools/other/ascii2nc/python_handler.cc index e2733a605e..d894ab6c64 100644 --- a/met/src/tools/other/ascii2nc/python_handler.cc +++ b/met/src/tools/other/ascii2nc/python_handler.cc @@ -27,13 +27,12 @@ using namespace std; static const char generic_python_wrapper [] = "generic_python"; -static const char generic_pickle_wrapper [] = "generic_pickle"; -static const char write_pickle_wrapper [] = "MET_BASE/wrappers/write_pickle_point.py"; +static const char write_tmp_ascii_wrapper[] = "MET_BASE/wrappers/write_tmp_point.py"; static const char list_name [] = "point_data"; -static const char pickle_base_name [] = "tmp_ascii2nc_pickle"; +static const char tmp_base_name [] = "tmp_ascii2nc"; //////////////////////////////////////////////////////////////////////// @@ -57,7 +56,7 @@ PythonHandler::PythonHandler(const string &program_name) : FileHandler(program_n { -use_pickle = false; +use_tmp_ascii = false; } @@ -82,13 +81,13 @@ for (j=1; j<(a.n()); ++j) { // j starts at one here, not zero } -use_pickle = false; +use_tmp_ascii = false; const char * c = getenv(user_python_path_env); if ( c ) { - use_pickle = true; + use_tmp_ascii = true; user_path_to_python = c; @@ -231,7 +230,7 @@ bool PythonHandler::readAsciiFiles(const vector< ConcatString > &ascii_filename_ bool status = false; -if ( use_pickle ) status = do_pickle (); +if ( use_tmp_ascii ) status = do_tmp_ascii (); else status = do_straight (); return ( status ); @@ -320,10 +319,10 @@ return ( true ); // - // wrapper usage: /path/to/python wrapper.py pickle_output_filename user_script_name [ user_script args ... ] + // wrapper usage: /path/to/python wrapper.py tmp_output_filename user_script_name [ user_script args ... ] // -bool PythonHandler::do_pickle() +bool PythonHandler::do_tmp_ascii() { @@ -331,7 +330,7 @@ int j; const int N = user_script_args.n(); ConcatString command; ConcatString path; -ConcatString pickle_path; +ConcatString tmp_ascii_path; const char * tmp_dir = 0; int status; @@ -345,15 +344,16 @@ if ( ! tmp_dir ) tmp_dir = default_tmp_dir; path << cs_erase << tmp_dir << '/' - << pickle_base_name; + << tmp_base_name; -pickle_path = make_temp_file_name(path.text(), 0); +tmp_ascii_path = make_temp_file_name(path.text(), 0); +tmp_ascii_path << ".txt"; command << cs_erase - << user_path_to_python << ' ' // user's path to python - << replace_path(write_pickle_wrapper) << ' ' // write_pickle.py - << pickle_path << ' ' // pickle output filename - << user_script_filename; // user's script name + << user_path_to_python << ' ' // user's path to python + << replace_path(write_tmp_ascii_wrapper) << ' ' // write_tmp_point.py + << tmp_ascii_path << ' ' // temporary ascii output filename + << user_script_filename; // user's script name for (j=0; j " + mlog << Error << "\nPythonHandler::do_tmp_ascii() -> " << "command \"" << command.text() << "\" failed ... status = " << status << "\n\n"; @@ -375,18 +375,20 @@ if ( status ) { ConcatString wrapper; -wrapper = generic_pickle_wrapper; +wrapper = generic_python_wrapper; Python3_Script script(wrapper.text()); -script.read_pickle(list_name, pickle_path.text()); +script.import_read_tmp_ascii_py(); -PyObject * obj = script.lookup(list_name); +PyObject * dobj = script.read_tmp_ascii(tmp_ascii_path.text()); + +PyObject * obj = script.lookup_ascii(list_name); if ( ! PyList_Check(obj) ) { - mlog << Error << "\nPythonHandler::do_pickle() -> " - << "pickle object is not a list!\n\n"; + mlog << Error << "\nPythonHandler::do_tmp_ascii() -> " + << "tmp ascii object is not a list!\n\n"; exit ( 1 ); @@ -398,7 +400,7 @@ load_python_obs(obj); // cleanup // -remove_temp_file(pickle_path); +remove_temp_file(tmp_ascii_path); // // done diff --git a/met/src/tools/other/ascii2nc/python_handler.h b/met/src/tools/other/ascii2nc/python_handler.h index abae8ddd5d..b0fb2ef492 100644 --- a/met/src/tools/other/ascii2nc/python_handler.h +++ b/met/src/tools/other/ascii2nc/python_handler.h @@ -50,9 +50,9 @@ class PythonHandler : public FileHandler static string getFormatString() { return "python"; } - bool use_pickle; + bool use_tmp_ascii; - ConcatString user_path_to_python; // if we're using pickle + ConcatString user_path_to_python; // if we're using temporary ascii ConcatString user_script_filename; @@ -68,15 +68,13 @@ class PythonHandler : public FileHandler virtual bool readAsciiFiles(const vector< ConcatString > &ascii_filename_list); - bool do_pickle (); - bool do_straight (); // straight-up python, no pickle + bool do_tmp_ascii(); + bool do_straight (); // straight-up python, no temporary ascii void load_python_obs(PyObject *); // python object is list of lists bool read_obs_from_script (const char * script_name, const char * variable_name); - - bool read_obs_from_pickle (const char * pickle_name, const char * variable_name); };