diff --git a/src/libcode/vx_data2d_nc_met/Makefile.am b/src/libcode/vx_data2d_nc_met/Makefile.am index 3aedaf4c59..f557d252ff 100644 --- a/src/libcode/vx_data2d_nc_met/Makefile.am +++ b/src/libcode/vx_data2d_nc_met/Makefile.am @@ -13,7 +13,7 @@ include ${top_srcdir}/Make-include noinst_LIBRARIES = libvx_data2d_nc_met.a libvx_data2d_nc_met_a_SOURCES = \ get_met_grid.cc get_met_grid.h \ - met_file.cc met_file.h \ + met_file.cc met_file.h met_file.hpp \ var_info_nc_met.cc var_info_nc_met.h \ data2d_nc_met.cc data2d_nc_met.h \ vx_data2d_nc_met.h diff --git a/src/libcode/vx_data2d_nc_met/Makefile.in b/src/libcode/vx_data2d_nc_met/Makefile.in index ebaa63426d..4c3c5965b5 100644 --- a/src/libcode/vx_data2d_nc_met/Makefile.in +++ b/src/libcode/vx_data2d_nc_met/Makefile.in @@ -341,7 +341,7 @@ MAINTAINERCLEANFILES = Makefile.in noinst_LIBRARIES = libvx_data2d_nc_met.a libvx_data2d_nc_met_a_SOURCES = \ get_met_grid.cc get_met_grid.h \ - met_file.cc met_file.h \ + met_file.cc met_file.h met_file.hpp \ var_info_nc_met.cc var_info_nc_met.h \ data2d_nc_met.cc data2d_nc_met.h \ vx_data2d_nc_met.h diff --git a/src/libcode/vx_data2d_nc_met/met_file.cc b/src/libcode/vx_data2d_nc_met/met_file.cc index 4aab77db9d..754a94b090 100644 --- a/src/libcode/vx_data2d_nc_met/met_file.cc +++ b/src/libcode/vx_data2d_nc_met/met_file.cc @@ -39,42 +39,6 @@ static const int max_met_args = 30; //////////////////////////////////////////////////////////////////////// -template - -void copy_nc_data_as_double(double *to_array, const T *from_array, - const int x_slot, const int y_slot, - const int nx, const int ny, - double missing_value, double fill_value) { - double value; - int x, y, offset, start_offset; - - offset = 0; - if (x_slot > y_slot) { - for (y=0; y + +void copy_nc_data_as_double(double *to_array, const T *from_array, + const int x_slot, const int y_slot, + const int nx, const int ny, + double missing_value, double fill_value) { + double value; + int x, y, offset, start_offset; + + offset = 0; + if (x_slot > y_slot) { + for (y=0; y -void load_numpy (void * buf, - const int Nx, const int Ny, - const int data_endian, - void (*shuf)(void *), - DataPlane & out) - - -{ - -bool need_swap = (shuf != 0) && (native_endian != data_endian); - -int j, x, y, r, c; -const int Nxy = Nx*Ny; -T * u = (T *) buf; -T value; - -for (j=0; j +void load_numpy (void * buf, + const int Nx, const int Ny, + const int data_endian, + void (*shuf)(void *), + DataPlane & out) + + +{ + +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j, x, y, r, c; +const int Nxy = Nx*Ny; +T * u = (T *) buf; +T value; + +for (j=0; j -bool get_att_num_value_(const NcAtt *att, T &att_val, int matching_type) { - bool status = false; - if (IS_VALID_NC_P(att)) { - int nc_type_id = GET_NC_TYPE_ID_P(att); - if (matching_type == nc_type_id) { - att->getValues(&att_val); - status = true; - } - else if (NC_CHAR == nc_type_id) { - string att_value; - att->getValues(att_value); - if (matching_type == NC_FLOAT) - att_val = atof(att_value.c_str()); - else if (matching_type == NC_DOUBLE) - att_val = (double)atof(att_value.c_str()); - else // if (matching_type == NC_INT) - att_val = atoi(att_value.c_str()); - status = true; - } - } - return(status); -} - -//////////////////////////////////////////////////////////////////////// - bool get_att_value(const NcAtt *att, ncbyte &att_val) { bool status = get_att_num_value_(att, att_val, NC_BYTE); return(status); @@ -489,35 +463,6 @@ bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, //////////////////////////////////////////////////////////////////////// -template -bool get_nc_att_value_(const NcVar *var, const ConcatString &att_name, - T &att_val, bool exit_on_error, - T bad_data, const char *caller_name) { - bool status = false; - - // Initialize - att_val = bad_data; - - // - // Retrieve the NetCDF variable attribute. - // - NcVarAtt *att = get_nc_att(var, att_name); - status = get_att_value((NcAtt *)att, att_val); - if (!status) { - mlog << Error << "\n" << caller_name - << get_log_msg_for_att(att, GET_SAFE_NC_NAME_P(var), att_name); - if (exit_on_error) { - if (att) delete att; - exit(1); - } - } - if (att) delete att; - - return(status); -} - -//////////////////////////////////////////////////////////////////////// - bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, int &att_val, bool exit_on_error) { static const char *method_name = "get_nc_att_value(NcVar,int) -> "; @@ -567,29 +512,6 @@ bool get_nc_att_value(const NcVarAtt *att, ConcatString &att_val) { //////////////////////////////////////////////////////////////////////// -template -bool get_nc_att_value_(const NcVarAtt *att, T &att_val, bool exit_on_error, - T bad_data, const char *caller_name) { - bool status = true; - - // Initialize - att_val = bad_data; - - // - // Retrieve the NetCDF variable attribute. - // - status = get_att_value((NcAtt *)att, att_val); - if (!status) { - mlog << Error << "\n" << caller_name << get_log_msg_for_att(att); - - if (exit_on_error) exit(1); - } - - return(status); -} - -//////////////////////////////////////////////////////////////////////// - bool get_nc_att_value(const NcVarAtt *att, int &att_val, bool exit_on_error) { static const char *method_name = "get_nc_att_value(NcVarAtt,int) -> "; bool status = get_nc_att_value_(att, att_val, exit_on_error, bad_data_int, method_name); @@ -767,37 +689,6 @@ bool get_global_att(const NcFile *nc, const ConcatString &att_name, //////////////////////////////////////////////////////////////////////// -template -bool get_global_att_value_(const NcFile *nc, const ConcatString& att_name, - T &att_val, T bad_data, bool error_out, const char *caller_name) { - bool status = false; - // Initialize - att_val = bad_data; - - NcGroupAtt *nc_att = get_nc_att(nc, att_name); - if (IS_VALID_NC_P(nc_att)) { - status = get_att_value((NcAtt *)nc_att, att_val); - string data_type = GET_NC_TYPE_NAME_P(nc_att); - if (error_out && !status) { - mlog << Error << caller_name - << "The data type \"" << data_type - << "\" for \"" << att_name << "\" does not match...\n\n"; - } - } - else if (error_out) { - mlog << Error << caller_name - << "can't find global NetCDF attribute \"" << att_name - << "\".\n\n"; - } - if (nc_att) delete nc_att; - // Check error_out status - if (error_out && !status) exit(1); - - return(status); -} - -//////////////////////////////////////////////////////////////////////// - bool get_global_att(const NcFile *nc, const ConcatString& att_name, int &att_val, bool error_out) { @@ -968,27 +859,6 @@ int get_var_names(NcFile *nc, StringArray *varNames) { //////////////////////////////////////////////////////////////////////// -template -bool get_var_att_num_(const NcVar *var, const ConcatString &att_name, - T &att_val, T bad_data) { - bool status = false; - - // Initialize - att_val = bad_data; - - NcVarAtt *att = get_nc_att(var, att_name); - // Look for a match - if (IS_VALID_NC_P(att)) { - att->getValues(&att_val); - status = true; - } - if (att) delete att; - - return(status); -} - -//////////////////////////////////////////////////////////////////////// - bool get_var_att_double(const NcVar *var, const ConcatString &att_name, double &att_val) { bool status = get_var_att_num_(var, att_name, att_val, bad_data_double); @@ -1041,40 +911,6 @@ void set_def_fill_value(unsigned short *val) { *val = (unsigned short)-1; } //////////////////////////////////////////////////////////////////////// -template -bool get_var_fill_value(const NcVar *var, T &att_val) { - bool found = false; - - NcVarAtt *att = get_nc_att(var, fill_value_att_name); - if (IS_INVALID_NC_P(att)) { - if (att) delete att; - att = get_nc_att(var, missing_value_att_name); - } - if (IS_VALID_NC_P(att)) { - att->getValues(&att_val); - found = true; - } - else set_def_fill_value(&att_val); - - if (att) delete att; - - return(found); -} - -//////////////////////////////////////////////////////////////////////// - -double get_var_fill_value(const NcVar *var) { - double v; - - if(!get_var_att_double(var, fill_value_att_name, v)) { - v = bad_data_double; - } - - return(v); -} - -//////////////////////////////////////////////////////////////////////// - bool get_var_grid_mapping(const NcVar *var, ConcatString &att_val) { return(get_nc_att_value(var, grid_mapping_att_name, att_val)); } @@ -1376,245 +1212,6 @@ float get_float_var(NcVar * var, const int index) { //////////////////////////////////////////////////////////////////////// -template -void apply_scale_factor_(T *data, const int cell_count, - double add_offset, double scale_factor, - const T nc_fill_value, const T met_fill_value, - bool has_fill_value, - const char *data_type, const char *var_name) { - const int debug_level = 7; - clock_t start_clock = clock(); - const char *method_name = "apply_scale_factor(T) "; - - if (cell_count > 0) { - int idx; - int positive_cnt = 0; - int unpacked_count = 0; - T min_value, max_value; - T raw_min_val, raw_max_val; - - idx = 0; - if (has_fill_value) { - for (; idx data[idx]) raw_min_val = data[idx]; - if (raw_max_val < data[idx]) raw_max_val = data[idx]; - data[idx] = (data[idx] * scale_factor) + add_offset; - if (data[idx] > 0) positive_cnt++; - if (min_value > data[idx]) min_value = data[idx]; - if (max_value < data[idx]) max_value = data[idx]; - unpacked_count++; - } - } - //cout << typeid(nc_fill_value).name(); - mlog << Debug(debug_level) << method_name << var_name - << "(" << typeid(data[0]).name() << "): unpacked data: count=" - << unpacked_count << " out of " << cell_count - << ", scale_factor=" << scale_factor<< " add_offset=" << add_offset - << ". FillValue(" << data_type << ")=" << nc_fill_value << "\n"; - mlog << Debug(debug_level) << method_name - << " data range [" << min_value << " - " << max_value - << "] raw data: [" << raw_min_val << " - " << raw_max_val - << "] Positive count: " << positive_cnt << "\n"; - } - mlog << Debug(debug_level) << method_name << " took " - << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; - return; -} - -//////////////////////////////////////////////////////////////////////// -// Note: -// - template _t reads data as is (do not apply no scale_factor and add_offset) -// - template _ reads data and applies scale_factor and add_offset. - -template -bool get_nc_data_t(NcVar *var, T *data) { - bool return_status = false; - - if (IS_VALID_NC_P(var)) { - var->getVar(data); - - return_status = true; - } - return(return_status); -} - -//////////////////////////////////////////////////////////////////////// - -template -bool get_nc_data_(NcVar *var, T *data, const T met_missing) { - //const char *method_name = "get_nc_data_() "; - - int data_size = get_data_size(var); - for (int idx1=0; idx1", GET_NC_NAME_P(var).c_str()); - } - } - return(return_status); -} - -//////////////////////////////////////////////////////////////////////// - -template -bool get_nc_data_(NcVar *var, T *data, T bad_data, const long *dims, const long *curs) { - bool return_status = false; - const char *method_name = "get_nc_data_(T, *dims, *curs) "; - - if (IS_VALID_NC_P(var)) { - std::vector start; - std::vector count; - - int data_size = 1; - int dimC = get_dim_count(var); - for (int idx = 0 ; idx < dimC; idx++) { - int dim_size = get_dim_size(var, idx); - if ((curs[idx]+dims[idx]) > dim_size) { - NcDim nc_dim = get_nc_dim(var, idx); - mlog << Error << "\n" << method_name << "The start offset and count (" - << curs[idx] << ", " << dims[idx] << ") exceeds the dimension[" - << idx << "] " << dim_size << " " - << (IS_VALID_NC(nc_dim) ? GET_NC_NAME(nc_dim) : " ") - << " for the variable " << GET_NC_NAME_P(var) << ".\n\n"; - exit(1); - } - - start.push_back((size_t)curs[idx]); - count.push_back((size_t)dims[idx]); - data_size *= dims[idx]; - } - - for (int idx1=0; idx1getVar(start, count, data); - return_status = true; - - //scale_factor and add_offset - if (has_add_offset_attr(var) || has_scale_factor_attr(var)) { - T nc_missing; - double add_offset = get_var_add_offset(var); - double scale_factor = get_var_scale_factor(var); - bool has_missing_attr = get_var_fill_value(var, nc_missing); - if (!has_missing_attr) nc_missing = bad_data; - apply_scale_factor_(data, data_size, add_offset, scale_factor, - nc_missing, bad_data, has_missing_attr, - "", GET_NC_NAME_P(var).c_str()); - } - } - return(return_status); -} - -//////////////////////////////////////////////////////////////////////// - -template -bool get_nc_data_(NcVar *var, T *data, T met_missing, const long dim, const long cur) { - bool return_status = false; - const char *method_name = "get_nc_data_(T, dim, cur) "; - for (int idx=0; idx start; - std::vector count; - start.push_back((size_t)cur); - count.push_back((size_t)dim); - int dim_size = get_dim_size(var, dim_idx); - if (0 >= dim_size) { - if ((cur > 0) || (dim > 1)) { - mlog << Error << "\n" << method_name << "The start offset and count (" - << cur << ", " << dim << ") should be (0, 1) because of no dimension at the variable " - << GET_NC_NAME_P(var) << ".\n\n"; - exit(1); - } - } - else if (((cur+dim) > dim_size) && (0 < dim_size)) { - NcDim nc_dim = get_nc_dim(var, dim_idx); - mlog << Error << "\n" << method_name << "The start offset and count (" - << cur << " + " << dim << ") exceeds the dimension " << dim_size << " " - << (IS_VALID_NC(nc_dim) ? GET_NC_NAME(nc_dim) : " ") - << " for the variable " << GET_NC_NAME_P(var) << ".\n\n"; - exit(1); - } - - // - // Retrieve the variable value from the NetCDF variable. - // Note: missing data was checked here - // - var->getVar(start, count, data); - return_status = true; - - //scale_factor and add_offset - if (has_add_offset_attr(var) || has_scale_factor_attr(var)) { - T nc_missing; - double add_offset = get_var_add_offset(var); - double scale_factor = get_var_scale_factor(var); - bool has_missing_attr = get_var_fill_value(var, nc_missing); - if (!has_missing_attr) nc_missing = met_missing; - apply_scale_factor_(data, dim, add_offset, scale_factor, - nc_missing, met_missing, has_missing_attr, - "", GET_NC_NAME_P(var).c_str()); - } - } - return(return_status); -} - -//////////////////////////////////////////////////////////////////////// -// read a single data - -template -bool get_nc_data_(NcVar *var, T *data, T bad_data, const long *curs) { - bool return_status = false; - const char *method_name = "get_nc_data_(*curs) "; - - if (IS_VALID_NC_P(var)) { - - int dimC = get_dim_count(var); - long dims[dimC]; - for (int idx = 0 ; idx < dimC; idx++) { - dims[idx] = 1; - } - - // Retrieve the NetCDF value from the NetCDF variable. - return_status = get_nc_data_(var, data, bad_data, dims, curs); - } - return(return_status); -} - -//////////////////////////////////////////////////////////////////////// - bool get_nc_data(NcVar *var, int *data, const long *curs) { bool return_status = get_nc_data_(var, data, bad_data_int, curs); @@ -1668,108 +1265,6 @@ bool get_nc_data(NcVar *var, short *data, const long *dims, const long *curs) { //////////////////////////////////////////////////////////////////////// -template -void copy_nc_data_t(NcVar *var, float *data, const T *packed_data, - const int cell_count, const char *data_type, - double add_offset, double scale_factor, - bool has_missing, T missing_value) { - clock_t start_clock = clock(); - const char *method_name = "copy_nc_data_t(float) "; - - if (cell_count > 0) { - int idx, first_idx; - float min_value, max_value; - bool do_scale_factor = has_scale_factor_attr(var) || has_add_offset_attr(var); - - idx = first_idx = 0; - if (do_scale_factor) { - int positive_cnt = 0; - int unpacked_count = 0; - T raw_min_val, raw_max_val; - - if (has_missing) { - for (; idx packed_data[idx]) raw_min_val = packed_data[idx]; - if (raw_max_val < packed_data[idx]) raw_max_val = packed_data[idx]; - data[idx] = ((float)packed_data[idx] * scale_factor) + add_offset; - if (data[idx] > 0) positive_cnt++; - if (min_value > data[idx]) min_value = data[idx]; - if (max_value < data[idx]) max_value = data[idx]; - unpacked_count++; - } - } - mlog << Debug(7) << method_name << GET_NC_NAME_P(var) - << " apply_scale_factor unpacked data: count=" - << unpacked_count << " out of " << cell_count - << ". FillValue(" << data_type << ")=" << missing_value << "\n"; - mlog << Debug(7) << method_name - << "data range [" << min_value << " - " << max_value - << "] raw data: [" << raw_min_val << " - " << raw_max_val - << "] Positive count: " << positive_cnt << "\n"; - } - else { - - if (has_missing) { - for (; idx data[idx]) min_value = data[idx]; - if (max_value < data[idx]) max_value = data[idx]; - } - } - mlog << Debug(7) << method_name << "data range [" << min_value - << " - " << max_value << "]\n"; - } - } - mlog << Debug(7) << method_name << "took " - << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; - return; -} - -//////////////////////////////////////////////////////////////////////// - -template -void copy_nc_data_(NcVar *var, float *data, const T *packed_data, - const int cell_count, const char *data_type, - double add_offset, double scale_factor) { - T missing_value; - bool has_missing = get_var_fill_value(var, missing_value); - copy_nc_data_t(var, data, packed_data, cell_count, data_type, - add_offset, scale_factor, has_missing, missing_value); - return; -} - -//////////////////////////////////////////////////////////////////////// - bool get_nc_data(NcVar *var, float *data) { bool return_status = false; clock_t start_clock = clock(); @@ -1971,103 +1466,6 @@ bool get_nc_data(NcFile *nc, const char *var_name, double *data, //////////////////////////////////////////////////////////////////////// -template -void copy_nc_data_t(NcVar *var, double *data, const T *packed_data, - const int cell_count, const char *data_type, - double add_offset, double scale_factor, - bool has_missing, T missing_value) { - int unpacked_count = 0; - const char *method_name = "copy_nc_data_t(double) "; - - if (cell_count > 0) { - int idx, first_idx; - double min_value, max_value; - bool do_scale_factor = has_scale_factor_attr(var) || has_add_offset_attr(var); - - idx = first_idx = 0; - if (do_scale_factor) { - int positive_cnt = 0; - T raw_min_val, raw_max_val; - - if (has_missing) { - // Skip missing values to find first min/max value - for (; idx packed_data[idx]) raw_min_val = packed_data[idx]; - if (raw_max_val < packed_data[idx]) raw_max_val = packed_data[idx]; - data[idx] = ((double)packed_data[idx] * scale_factor) + add_offset; - if (data[idx] > 0) positive_cnt++; - if (min_value > data[idx]) min_value = data[idx]; - if (max_value < data[idx]) max_value = data[idx]; - unpacked_count++; - } - } - mlog << Debug(7) << method_name << GET_NC_NAME_P(var) - << " apply_scale_factor unpacked data: count=" - << unpacked_count << " out of " << cell_count - << ". FillValue(" << data_type << ")=" << missing_value - << " data range [" << min_value << " - " << max_value - << "] raw data: [" << raw_min_val << " - " << raw_max_val - << "] Positive count: " << positive_cnt << "\n"; - } - else { - if (has_missing) { - for (; idx data[idx]) min_value = data[idx]; - if (max_value < data[idx]) max_value = data[idx]; - } - } - mlog << Debug(7) << method_name << "data range [" << min_value - << " - " << max_value << "]\n"; - } - } -} - -//////////////////////////////////////////////////////////////////////// - -template -void copy_nc_data_(NcVar *var, double *data, const T *packed_data, - const int cell_count, const char *data_type, - double add_offset, double scale_factor) { - T missing_value; - bool has_missing = get_var_fill_value(var, missing_value); - copy_nc_data_t(var, data, packed_data, cell_count, data_type, - add_offset, scale_factor, has_missing, missing_value); - return; -} - -//////////////////////////////////////////////////////////////////////// - bool get_nc_data(NcVar *var, double *data) { bool return_status = false; static const char *method_name = "get_nc_data(NcVar *, double *) "; @@ -2536,22 +1934,6 @@ int get_nc_string_length(NcFile *nc_file, NcVar var, const char *var_name) { //////////////////////////////////////////////////////////////////////// -template -bool put_nc_data_T(NcVar *var, const T data, long offset0, long offset1, long offset2) { - vector offsets; - offsets.push_back((size_t)offset0); - if (0 <= offset1) { - offsets.push_back((size_t)offset1); - } - if (0 <= offset2) { - offsets.push_back((size_t)offset2); - } - var->putVar(offsets, data); - return true; -} - -//////////////////////////////////////////////////////////////////////// - bool put_nc_data(NcVar *var, const int data, long offset0, long offset1, long offset2) { return put_nc_data_T(var, data, offset0, offset1, offset2); } @@ -2617,22 +1999,6 @@ bool put_nc_data(NcVar *var, const ncbyte *data ) { //////////////////////////////////////////////////////////////////////// -template -bool put_nc_data_T(NcVar *var, const T *data, const long length, const long offset) { - vector offsets, counts; - int dim_count = get_dim_count(var); - offsets.push_back(offset); - if (dim_count >= 2) { - offsets.push_back(0); - counts.push_back(1); - } - counts.push_back(length); - var->putVar(offsets, counts, data); - return true; -} - -//////////////////////////////////////////////////////////////////////// - bool put_nc_data(NcVar *var, const int *data, const long length, const long offset) { put_nc_data_T(var, data, length, offset); return true; @@ -2668,22 +2034,6 @@ bool put_nc_data(NcVar *var, const ncbyte *data, const long length, const long o //////////////////////////////////////////////////////////////////////// -template -bool put_nc_data_T(NcVar *var, const T *data , const long *lengths, const long *offsets) { - int dim = get_dim_count(var); - vector nc_offsets, counts; - for (int idx = 0 ; idx < dim; idx++) { - nc_offsets.push_back(offsets[idx]); - } - for (int idx = 0 ; idx < dim; idx++) { - counts.push_back(lengths[idx]); - } - var->putVar(nc_offsets, counts, data); - return true; -} - -//////////////////////////////////////////////////////////////////////// - bool put_nc_data(NcVar *var, const float *data , const long *lengths, const long *offsets) { put_nc_data_T(var, data , lengths, offsets); return true; @@ -2705,28 +2055,6 @@ bool put_nc_data(NcVar *var, const int *data , const long *lengths, const long * //////////////////////////////////////////////////////////////////////// -template -bool put_nc_data_T_with_dims(NcVar *var, const T *data, - const long len0, const long len1, const long len2) { - vector offsets, counts; - if (0 < len0) { - offsets.push_back(0); - counts.push_back(len0); - } - if (0 < len1) { - offsets.push_back(0); - counts.push_back(len1); - } - if (0 < len2) { - offsets.push_back(0); - counts.push_back(len2); - } - var->putVar(offsets, counts, data); - return true; -} - -//////////////////////////////////////////////////////////////////////// - bool put_nc_data_with_dims(NcVar *var, const int *data, const int len0, const int len1, const int len2) { return put_nc_data_with_dims(var, data, (long)len0, (long)len1, (long)len2); diff --git a/src/libcode/vx_nc_util/nc_utils.h b/src/libcode/vx_nc_util/nc_utils.h index 9db75b08ea..58fed1d8c3 100644 --- a/src/libcode/vx_nc_util/nc_utils.h +++ b/src/libcode/vx_nc_util/nc_utils.h @@ -226,7 +226,6 @@ extern bool get_var_att_double(const NcVar *, const ConcatString &, double &); template extern bool get_var_fill_value(const NcVar *var, T &att_val); extern bool get_var_axis(const NcVar *var, ConcatString &att_val); -extern double get_var_fill_value(const NcVar *); extern bool get_var_grid_mapping(const NcVar *var, ConcatString &att_val); extern bool get_var_grid_mapping_name(const NcVar *var, ConcatString &att_val); extern bool get_var_long_name(const NcVar *, ConcatString &); @@ -371,6 +370,10 @@ extern void parse_cf_time_string(const char *str, unixtime &ref_ut, //////////////////////////////////////////////////////////////////////// +#include "nc_utils.hpp" + +//////////////////////////////////////////////////////////////////////// + #endif /* __NC_UTILS_H__ */ //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_nc_util/nc_utils.hpp b/src/libcode/vx_nc_util/nc_utils.hpp new file mode 100644 index 0000000000..f66b665a29 --- /dev/null +++ b/src/libcode/vx_nc_util/nc_utils.hpp @@ -0,0 +1,710 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2022 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +#ifndef __NC_UTILS_HPP__ +#define __NC_UTILS_HPP__ + +//////////////////////////////////////////////////////////////////////// + +extern bool get_att_value(const NcAtt *att, int &att_val); +extern bool get_att_value(const NcAtt *att, ConcatString &value); +extern bool get_att_value(const NcAtt *att, ncbyte &att_val); +extern bool get_att_value(const NcAtt *att, short &att_val); +extern bool get_att_value(const NcAtt *att, int &att_val); +extern bool get_att_value(const NcAtt *att, unsigned int &att_val); +extern bool get_att_value(const NcAtt *att, float &att_val); +extern bool get_att_value(const NcAtt *att, double &att_val); +extern ConcatString get_log_msg_for_att(const NcVarAtt *att); +extern ConcatString get_log_msg_for_att(const NcVarAtt *att, string var_name, + const ConcatString att_name); +extern double get_var_add_offset(const NcVar *var); +extern double get_var_scale_factor(const NcVar *var); +extern bool has_add_offset_attr(NcVar *var); +extern bool has_scale_factor_attr(NcVar *var); +extern void set_def_fill_value(ncbyte *val); +extern void set_def_fill_value(char *val); +extern void set_def_fill_value(double *val); +extern void set_def_fill_value(float *val); +extern void set_def_fill_value(int *val); +extern void set_def_fill_value(long *val); +extern void set_def_fill_value(short *val); +extern void set_def_fill_value(long long *val); +extern void set_def_fill_value(unsigned char *val); +extern void set_def_fill_value(unsigned int *val); +extern void set_def_fill_value(unsigned long *val); +extern void set_def_fill_value(unsigned long long *val); +extern void set_def_fill_value(unsigned short *val); + +//////////////////////////////////////////////////////////////////////// + +template +bool get_att_num_value_(const NcAtt *att, T &att_val, int matching_type) { + bool status = false; + if (IS_VALID_NC_P(att)) { + int nc_type_id = GET_NC_TYPE_ID_P(att); + if (matching_type == nc_type_id) { + att->getValues(&att_val); + status = true; + } + else if (NC_CHAR == nc_type_id) { + string att_value; + att->getValues(att_value); + if (matching_type == NC_FLOAT) + att_val = atof(att_value.c_str()); + else if (matching_type == NC_DOUBLE) + att_val = (double)atof(att_value.c_str()); + else // if (matching_type == NC_INT) + att_val = atoi(att_value.c_str()); + status = true; + } + } + return(status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_nc_att_value_(const NcVar *var, const ConcatString &att_name, + T &att_val, bool exit_on_error, + T bad_data, const char *caller_name) { + bool status = false; + + // Initialize + att_val = bad_data; + + // + // Retrieve the NetCDF variable attribute. + // + NcVarAtt *att = get_nc_att(var, att_name); + status = get_att_value((NcAtt *)att, att_val); + if (!status) { + mlog << Error << "\n" << caller_name + << get_log_msg_for_att(att, GET_SAFE_NC_NAME_P(var), att_name); + if (exit_on_error) { + if (att) delete att; + exit(1); + } + } + if (att) delete att; + + return(status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_nc_att_value_(const NcVarAtt *att, T &att_val, bool exit_on_error, + T bad_data, const char *caller_name) { + bool status = true; + + // Initialize + att_val = bad_data; + + // + // Retrieve the NetCDF variable attribute. + // + status = get_att_value((NcAtt *)att, att_val); + if (!status) { + mlog << Error << "\n" << caller_name << get_log_msg_for_att(att); + + if (exit_on_error) exit(1); + } + + return(status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_global_att_value_(const NcFile *nc, const ConcatString& att_name, + T &att_val, T bad_data, bool error_out, const char *caller_name) { + bool status = false; + // Initialize + att_val = bad_data; + + NcGroupAtt *nc_att = get_nc_att(nc, att_name); + if (IS_VALID_NC_P(nc_att)) { + status = get_att_value((NcAtt *)nc_att, att_val); + string data_type = GET_NC_TYPE_NAME_P(nc_att); + if (error_out && !status) { + mlog << Error << caller_name + << "The data type \"" << data_type + << "\" for \"" << att_name << "\" does not match...\n\n"; + } + } + else if (error_out) { + mlog << Error << caller_name + << "can't find global NetCDF attribute \"" << att_name + << "\".\n\n"; + } + if (nc_att) delete nc_att; + // Check error_out status + if (error_out && !status) exit(1); + + return(status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_var_att_num_(const NcVar *var, const ConcatString &att_name, + T &att_val, T bad_data) { + bool status = false; + + // Initialize + att_val = bad_data; + + NcVarAtt *att = get_nc_att(var, att_name); + // Look for a match + if (IS_VALID_NC_P(att)) { + att->getValues(&att_val); + status = true; + } + if (att) delete att; + + return(status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_var_fill_value(const NcVar *var, T &att_val) { + bool found = false; + + NcVarAtt *att = get_nc_att(var, fill_value_att_name); + if (IS_INVALID_NC_P(att)) { + if (att) delete att; + att = get_nc_att(var, missing_value_att_name); + } + if (IS_VALID_NC_P(att)) { + att->getValues(&att_val); + found = true; + } + else set_def_fill_value(&att_val); + + if (att) delete att; + + return(found); +} + +//////////////////////////////////////////////////////////////////////// + +template +void apply_scale_factor_(T *data, const int cell_count, + double add_offset, double scale_factor, + const T nc_fill_value, const T met_fill_value, + bool has_fill_value, + const char *data_type, const char *var_name) { + const int debug_level = 7; + clock_t start_clock = clock(); + const char *method_name = "apply_scale_factor(T) "; + + if (cell_count > 0) { + int idx; + int positive_cnt = 0; + int unpacked_count = 0; + T min_value, max_value; + T raw_min_val, raw_max_val; + + idx = 0; + if (has_fill_value) { + for (; idx data[idx]) raw_min_val = data[idx]; + if (raw_max_val < data[idx]) raw_max_val = data[idx]; + data[idx] = (data[idx] * scale_factor) + add_offset; + if (data[idx] > 0) positive_cnt++; + if (min_value > data[idx]) min_value = data[idx]; + if (max_value < data[idx]) max_value = data[idx]; + unpacked_count++; + } + } + //cout << typeid(nc_fill_value).name(); + mlog << Debug(debug_level) << method_name << var_name + << "(" << typeid(data[0]).name() << "): unpacked data: count=" + << unpacked_count << " out of " << cell_count + << ", scale_factor=" << scale_factor<< " add_offset=" << add_offset + << ". FillValue(" << data_type << ")=" << nc_fill_value << "\n"; + mlog << Debug(debug_level) << method_name + << " data range [" << min_value << " - " << max_value + << "] raw data: [" << raw_min_val << " - " << raw_max_val + << "] Positive count: " << positive_cnt << "\n"; + } + mlog << Debug(debug_level) << method_name << " took " + << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; + return; +} + +//////////////////////////////////////////////////////////////////////// +// Note: +// - template _t reads data as is (do not apply no scale_factor and add_offset) +// - template _ reads data and applies scale_factor and add_offset. + +template +bool get_nc_data_t(NcVar *var, T *data) { + bool return_status = false; + + if (IS_VALID_NC_P(var)) { + var->getVar(data); + + return_status = true; + } + return(return_status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_nc_data_(NcVar *var, T *data, const T met_missing) { + //const char *method_name = "get_nc_data_() "; + + int data_size = get_data_size(var); + for (int idx1=0; idx1", GET_NC_NAME_P(var).c_str()); + } + } + return(return_status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_nc_data_(NcVar *var, T *data, T bad_data, const long *dims, const long *curs) { + bool return_status = false; + const char *method_name = "get_nc_data_(T, *dims, *curs) "; + + if (IS_VALID_NC_P(var)) { + std::vector start; + std::vector count; + + int data_size = 1; + int dimC = get_dim_count(var); + for (int idx = 0 ; idx < dimC; idx++) { + int dim_size = get_dim_size(var, idx); + if ((curs[idx]+dims[idx]) > dim_size) { + NcDim nc_dim = get_nc_dim(var, idx); + mlog << Error << "\n" << method_name << "The start offset and count (" + << curs[idx] << ", " << dims[idx] << ") exceeds the dimension[" + << idx << "] " << dim_size << " " + << (IS_VALID_NC(nc_dim) ? GET_NC_NAME(nc_dim) : " ") + << " for the variable " << GET_NC_NAME_P(var) << ".\n\n"; + exit(1); + } + + start.push_back((size_t)curs[idx]); + count.push_back((size_t)dims[idx]); + data_size *= dims[idx]; + } + + for (int idx1=0; idx1getVar(start, count, data); + return_status = true; + + //scale_factor and add_offset + if (has_add_offset_attr(var) || has_scale_factor_attr(var)) { + T nc_missing; + double add_offset = get_var_add_offset(var); + double scale_factor = get_var_scale_factor(var); + bool has_missing_attr = get_var_fill_value(var, nc_missing); + if (!has_missing_attr) nc_missing = bad_data; + apply_scale_factor_(data, data_size, add_offset, scale_factor, + nc_missing, bad_data, has_missing_attr, + "", GET_NC_NAME_P(var).c_str()); + } + } + return(return_status); +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_nc_data_(NcVar *var, T *data, T met_missing, const long dim, const long cur) { + bool return_status = false; + const char *method_name = "get_nc_data_(T, dim, cur) "; + for (int idx=0; idx start; + std::vector count; + start.push_back((size_t)cur); + count.push_back((size_t)dim); + int dim_size = get_dim_size(var, dim_idx); + if (0 >= dim_size) { + if ((cur > 0) || (dim > 1)) { + mlog << Error << "\n" << method_name << "The start offset and count (" + << cur << ", " << dim << ") should be (0, 1) because of no dimension at the variable " + << GET_NC_NAME_P(var) << ".\n\n"; + exit(1); + } + } + else if (((cur+dim) > dim_size) && (0 < dim_size)) { + NcDim nc_dim = get_nc_dim(var, dim_idx); + mlog << Error << "\n" << method_name << "The start offset and count (" + << cur << " + " << dim << ") exceeds the dimension " << dim_size << " " + << (IS_VALID_NC(nc_dim) ? GET_NC_NAME(nc_dim) : " ") + << " for the variable " << GET_NC_NAME_P(var) << ".\n\n"; + exit(1); + } + + // + // Retrieve the variable value from the NetCDF variable. + // Note: missing data was checked here + // + var->getVar(start, count, data); + return_status = true; + + //scale_factor and add_offset + if (has_add_offset_attr(var) || has_scale_factor_attr(var)) { + T nc_missing; + double add_offset = get_var_add_offset(var); + double scale_factor = get_var_scale_factor(var); + bool has_missing_attr = get_var_fill_value(var, nc_missing); + if (!has_missing_attr) nc_missing = met_missing; + apply_scale_factor_(data, dim, add_offset, scale_factor, + nc_missing, met_missing, has_missing_attr, + "", GET_NC_NAME_P(var).c_str()); + } + } + return(return_status); +} + +//////////////////////////////////////////////////////////////////////// +// read a single data + +template +bool get_nc_data_(NcVar *var, T *data, T bad_data, const long *curs) { + bool return_status = false; + const char *method_name = "get_nc_data_(*curs) "; + + if (IS_VALID_NC_P(var)) { + + int dimC = get_dim_count(var); + long dims[dimC]; + for (int idx = 0 ; idx < dimC; idx++) { + dims[idx] = 1; + } + + // Retrieve the NetCDF value from the NetCDF variable. + return_status = get_nc_data_(var, data, bad_data, dims, curs); + } + return(return_status); +} + +//////////////////////////////////////////////////////////////////////// + +template +void copy_nc_data_t(NcVar *var, float *data, const T *packed_data, + const int cell_count, const char *data_type, + double add_offset, double scale_factor, + bool has_missing, T missing_value) { + clock_t start_clock = clock(); + const char *method_name = "copy_nc_data_t(float) "; + + if (cell_count > 0) { + int idx, first_idx; + float min_value, max_value; + bool do_scale_factor = has_scale_factor_attr(var) || has_add_offset_attr(var); + + idx = first_idx = 0; + if (do_scale_factor) { + int positive_cnt = 0; + int unpacked_count = 0; + T raw_min_val, raw_max_val; + + if (has_missing) { + for (; idx packed_data[idx]) raw_min_val = packed_data[idx]; + if (raw_max_val < packed_data[idx]) raw_max_val = packed_data[idx]; + data[idx] = ((float)packed_data[idx] * scale_factor) + add_offset; + if (data[idx] > 0) positive_cnt++; + if (min_value > data[idx]) min_value = data[idx]; + if (max_value < data[idx]) max_value = data[idx]; + unpacked_count++; + } + } + mlog << Debug(7) << method_name << GET_NC_NAME_P(var) + << " apply_scale_factor unpacked data: count=" + << unpacked_count << " out of " << cell_count + << ". FillValue(" << data_type << ")=" << missing_value << "\n"; + mlog << Debug(7) << method_name + << "data range [" << min_value << " - " << max_value + << "] raw data: [" << raw_min_val << " - " << raw_max_val + << "] Positive count: " << positive_cnt << "\n"; + } + else { + + if (has_missing) { + for (; idx data[idx]) min_value = data[idx]; + if (max_value < data[idx]) max_value = data[idx]; + } + } + mlog << Debug(7) << method_name << "data range [" << min_value + << " - " << max_value << "]\n"; + } + } + mlog << Debug(7) << method_name << "took " + << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; + return; +} + +//////////////////////////////////////////////////////////////////////// + +template +void copy_nc_data_(NcVar *var, float *data, const T *packed_data, + const int cell_count, const char *data_type, + double add_offset, double scale_factor) { + T missing_value; + bool has_missing = get_var_fill_value(var, missing_value); + copy_nc_data_t(var, data, packed_data, cell_count, data_type, + add_offset, scale_factor, has_missing, missing_value); + return; +} + +//////////////////////////////////////////////////////////////////////// + +template +void copy_nc_data_t(NcVar *var, double *data, const T *packed_data, + const int cell_count, const char *data_type, + double add_offset, double scale_factor, + bool has_missing, T missing_value) { + int unpacked_count = 0; + const char *method_name = "copy_nc_data_t(double) "; + + if (cell_count > 0) { + int idx, first_idx; + double min_value, max_value; + bool do_scale_factor = has_scale_factor_attr(var) || has_add_offset_attr(var); + + idx = first_idx = 0; + if (do_scale_factor) { + int positive_cnt = 0; + T raw_min_val, raw_max_val; + + if (has_missing) { + // Skip missing values to find first min/max value + for (; idx packed_data[idx]) raw_min_val = packed_data[idx]; + if (raw_max_val < packed_data[idx]) raw_max_val = packed_data[idx]; + data[idx] = ((double)packed_data[idx] * scale_factor) + add_offset; + if (data[idx] > 0) positive_cnt++; + if (min_value > data[idx]) min_value = data[idx]; + if (max_value < data[idx]) max_value = data[idx]; + unpacked_count++; + } + } + mlog << Debug(7) << method_name << GET_NC_NAME_P(var) + << " apply_scale_factor unpacked data: count=" + << unpacked_count << " out of " << cell_count + << ". FillValue(" << data_type << ")=" << missing_value + << " data range [" << min_value << " - " << max_value + << "] raw data: [" << raw_min_val << " - " << raw_max_val + << "] Positive count: " << positive_cnt << "\n"; + } + else { + if (has_missing) { + for (; idx data[idx]) min_value = data[idx]; + if (max_value < data[idx]) max_value = data[idx]; + } + } + mlog << Debug(7) << method_name << "data range [" << min_value + << " - " << max_value << "]\n"; + } + } +} + +//////////////////////////////////////////////////////////////////////// + +template +void copy_nc_data_(NcVar *var, double *data, const T *packed_data, + const int cell_count, const char *data_type, + double add_offset, double scale_factor) { + T missing_value; + bool has_missing = get_var_fill_value(var, missing_value); + copy_nc_data_t(var, data, packed_data, cell_count, data_type, + add_offset, scale_factor, has_missing, missing_value); + return; +} + +//////////////////////////////////////////////////////////////////////// + +template +bool put_nc_data_T(NcVar *var, const T data, long offset0, long offset1, long offset2) { + vector offsets; + offsets.push_back((size_t)offset0); + if (0 <= offset1) { + offsets.push_back((size_t)offset1); + } + if (0 <= offset2) { + offsets.push_back((size_t)offset2); + } + var->putVar(offsets, data); + return true; +} + +//////////////////////////////////////////////////////////////////////// + +template +bool put_nc_data_T(NcVar *var, const T *data, const long length, const long offset) { + vector offsets, counts; + int dim_count = get_dim_count(var); + offsets.push_back(offset); + if (dim_count >= 2) { + offsets.push_back(0); + counts.push_back(1); + } + counts.push_back(length); + var->putVar(offsets, counts, data); + return true; +} + +//////////////////////////////////////////////////////////////////////// + +template +bool put_nc_data_T(NcVar *var, const T *data , const long *lengths, const long *offsets) { + int dim = get_dim_count(var); + vector nc_offsets, counts; + for (int idx = 0 ; idx < dim; idx++) { + nc_offsets.push_back(offsets[idx]); + } + for (int idx = 0 ; idx < dim; idx++) { + counts.push_back(lengths[idx]); + } + var->putVar(nc_offsets, counts, data); + return true; +} + +//////////////////////////////////////////////////////////////////////// + +template +bool put_nc_data_T_with_dims(NcVar *var, const T *data, + const long len0, const long len1, const long len2) { + vector offsets, counts; + if (0 < len0) { + offsets.push_back(0); + counts.push_back(len0); + } + if (0 < len1) { + offsets.push_back(0); + counts.push_back(len1); + } + if (0 < len2) { + offsets.push_back(0); + counts.push_back(len2); + } + var->putVar(offsets, counts, data); + return true; +} + + +//////////////////////////////////////////////////////////////////////// + +#endif /* __NC_UTILS_HPP__ */ + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_pointdata_python/Makefile.am b/src/libcode/vx_pointdata_python/Makefile.am index c48c07e2df..5579afc6fe 100644 --- a/src/libcode/vx_pointdata_python/Makefile.am +++ b/src/libcode/vx_pointdata_python/Makefile.am @@ -13,7 +13,7 @@ include ${top_srcdir}/Make-include noinst_LIBRARIES = libvx_pointdata_python.a libvx_pointdata_python_a_SOURCES = \ pointdata_python.h pointdata_python.cc \ - pointdata_from_array.h pointdata_from_array.cc \ - python_pointdata.h python_pointdata.cc + pointdata_from_array.h pointdata_from_array.cc pointdata_from_array.hpp \ + python_pointdata.h python_pointdata.cc python_pointdata.hpp libvx_pointdata_python_a_CPPFLAGS = ${MET_CPPFLAGS} -I../vx_python2_utils ${MET_PYTHON_CC} $(MET_PYTHON_LD) diff --git a/src/libcode/vx_pointdata_python/Makefile.in b/src/libcode/vx_pointdata_python/Makefile.in index 8af9428f28..d45f6a82da 100644 --- a/src/libcode/vx_pointdata_python/Makefile.in +++ b/src/libcode/vx_pointdata_python/Makefile.in @@ -340,8 +340,8 @@ MAINTAINERCLEANFILES = Makefile.in noinst_LIBRARIES = libvx_pointdata_python.a libvx_pointdata_python_a_SOURCES = \ pointdata_python.h pointdata_python.cc \ - pointdata_from_array.h pointdata_from_array.cc \ - python_pointdata.h python_pointdata.cc + pointdata_from_array.h pointdata_from_array.cc pointdata_from_array.hpp \ + python_pointdata.h python_pointdata.cc python_pointdata.hpp libvx_pointdata_python_a_CPPFLAGS = ${MET_CPPFLAGS} -I../vx_python2_utils ${MET_PYTHON_CC} $(MET_PYTHON_LD) all: all-am diff --git a/src/libcode/vx_pointdata_python/pointdata_from_array.cc b/src/libcode/vx_pointdata_python/pointdata_from_array.cc index 3de0fdf9d2..ae196e40b1 100644 --- a/src/libcode/vx_pointdata_python/pointdata_from_array.cc +++ b/src/libcode/vx_pointdata_python/pointdata_from_array.cc @@ -19,160 +19,6 @@ //////////////////////////////////////////////////////////////////////// -static const int api_debug_level = 11; - -//////////////////////////////////////////////////////////////////////// - - -template -void load_numpy (void * buf, - const int n, - const int data_endian, - void (*shuf)(void *), - float * out) -{ - -bool need_swap = (shuf != 0) && (native_endian != data_endian); - -int j; -T * u = (T *) buf; -T value; - -for (j=0; j -void load_numpy (void * buf, - const int n, - const int data_endian, - void (*shuf)(void *), - int * out) -{ - -const char *method_name = "load_numpy(int *) "; -bool need_swap = (shuf != 0) && (native_endian != data_endian); - -int j; -T * u = (T *) buf; -T value; - -for (j=0; j -void load_numpy_int (void * buf, - const int n, - const int data_endian, - void (*shuf)(void *), - IntArray *out) -{ - -const char *method_name = "load_numpy_int(IntArray *) "; -bool need_swap = (shuf != 0) && (native_endian != data_endian); - -int j; -T * u = (T *) buf; -T value; - -out->extend(n); - -for (j=0; jadd((int)value); - - mlog << Debug(api_debug_level) << method_name << " [" << j << "] value=" << value << "\n"; -} // for j - - - -return; - -} - - -//////////////////////////////////////////////////////////////////////// - - -template -void load_numpy_num (void * buf, - const int n, - const int data_endian, - void (*shuf)(void *), - NumArray *out) -{ - -const char *method_name = "load_numpy_num(NumArray *) "; -bool need_swap = (shuf != 0) && (native_endian != data_endian); - -int j; -T * u = (T *) buf; -T value; - -out->extend(n); - -for (j=0; jadd((float)value); - - mlog << Debug(api_debug_level) << method_name << "[" << j << "] value=" << value << "\n"; -} // for j - - - -return; - -} - //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_pointdata_python/pointdata_from_array.h b/src/libcode/vx_pointdata_python/pointdata_from_array.h index 7538ef51bc..2e6bc12d21 100644 --- a/src/libcode/vx_pointdata_python/pointdata_from_array.h +++ b/src/libcode/vx_pointdata_python/pointdata_from_array.h @@ -49,6 +49,9 @@ extern bool pointdata_from_python_list(PyObject *data_array, NumArray *data_out) //////////////////////////////////////////////////////////////////////// +#include "pointdata_from_array.hpp" + +//////////////////////////////////////////////////////////////////////// #endif /* __MET_POINTDATA_FROM_NUMPY_ARRAY_H__ */ diff --git a/src/libcode/vx_pointdata_python/pointdata_from_array.hpp b/src/libcode/vx_pointdata_python/pointdata_from_array.hpp new file mode 100644 index 0000000000..f9bf3cdb8c --- /dev/null +++ b/src/libcode/vx_pointdata_python/pointdata_from_array.hpp @@ -0,0 +1,176 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2022 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + + +#ifndef __MET_POINTDATA_FROM_ARRAY_HPP__ +#define __MET_POINTDATA_FROM_ARRAY_HPP__ + + +//////////////////////////////////////////////////////////////////////// + +static const int api_debug_level = 11; + +//////////////////////////////////////////////////////////////////////// + + +template +void load_numpy (void * buf, + const int n, + const int data_endian, + void (*shuf)(void *), + float * out) +{ + +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j; +T * u = (T *) buf; +T value; + +for (j=0; j +void load_numpy (void * buf, + const int n, + const int data_endian, + void (*shuf)(void *), + int * out) +{ + +const char *method_name = "load_numpy(int *) "; +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j; +T * u = (T *) buf; +T value; + +for (j=0; j +void load_numpy_int (void * buf, + const int n, + const int data_endian, + void (*shuf)(void *), + IntArray *out) +{ + +const char *method_name = "load_numpy_int(IntArray *) "; +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j; +T * u = (T *) buf; +T value; + +out->extend(n); + +for (j=0; jadd((int)value); + + mlog << Debug(api_debug_level) << method_name << " [" << j << "] value=" << value << "\n"; +} // for j + + + +return; + +} + + +//////////////////////////////////////////////////////////////////////// + + +template +void load_numpy_num (void * buf, + const int n, + const int data_endian, + void (*shuf)(void *), + NumArray *out) +{ + +const char *method_name = "load_numpy_num(NumArray *) "; +bool need_swap = (shuf != 0) && (native_endian != data_endian); + +int j; +T * u = (T *) buf; +T value; + +out->extend(n); + +for (j=0; jadd((float)value); + + mlog << Debug(api_debug_level) << method_name << "[" << j << "] value=" << value << "\n"; +} // for j + + + +return; + +} + +#endif /* __MET_POINTDATA_FROM_NUMPY_ARRAY_HPP__ */ + + +//////////////////////////////////////////////////////////////////////// + diff --git a/src/libcode/vx_pointdata_python/python_pointdata.cc b/src/libcode/vx_pointdata_python/python_pointdata.cc index 8a85848274..e99cc49596 100644 --- a/src/libcode/vx_pointdata_python/python_pointdata.cc +++ b/src/libcode/vx_pointdata_python/python_pointdata.cc @@ -26,59 +26,6 @@ static bool straight_python_point_data(const char * script_name, int script_argc, char ** script_argv, const bool use_xarray, MetPointDataPython &met_pd_out); -//////////////////////////////////////////////////////////////////////// - -template -static void set_array_from_python(PyObject *python_data, const char *python_key, T *out, bool required=true) { - const char *method_name = "set_array_from_python(T *) -> "; - PyObject *numpy_array_obj = PyDict_GetItemString (python_data, python_key); - if (numpy_array_obj) { - bool status = false; - ConcatString py_type_name = Py_TYPE(numpy_array_obj)->tp_name; - if ("numpy.ndarray" == py_type_name || "MaskedArray" == py_type_name ){ - Python3_Numpy np; - np.set(numpy_array_obj); - pointdata_from_np_array(np, out); - status = true; - } - else if (PyList_Check(numpy_array_obj)) { - status = pointdata_from_python_list(numpy_array_obj, out); - if(!status) { - if (PyFloat_Check(PyList_GetItem(numpy_array_obj, 0))) { - mlog << Error << "\n" << method_name - << "Only int type is supported at python list for " << python_key << "." - << " Check the data type from python and consider using numpy for python embedding\n\n"; - } - else { - mlog << Error << "\n" << method_name - << "Only int/float type is supported at python list for " << python_key << "." - << " Check the data type from python and consider using numpy for python embedding\n\n"; - } - } - } - else { - mlog << Error << "\n" << method_name - << "Not getting the point data by the key (" << python_key << ") from python object\n" - << " The python type \"" << py_type_name << "\" is not supported\n\n"; - } - if (status) { - mlog << Debug(7) << method_name - << "get the point data for " << python_key << " from python object\n"; - } - else exit (1); - } - else { - if (required) { - mlog << Error << "\n" << method_name - << "error getting the point data by the key (" << python_key << ") from python object\n\n"; - exit (1); - } - else mlog << Debug(3) << method_name - << "not exists the point data (" << python_key << ") from python object\n"; - } -} - - //////////////////////////////////////////////////////////////////////// static void set_str_array_from_python(PyObject *python_data, const char *python_key, StringArray *out) { diff --git a/src/libcode/vx_pointdata_python/python_pointdata.h b/src/libcode/vx_pointdata_python/python_pointdata.h index 54cd3ede9b..34005715e4 100644 --- a/src/libcode/vx_pointdata_python/python_pointdata.h +++ b/src/libcode/vx_pointdata_python/python_pointdata.h @@ -9,8 +9,8 @@ //////////////////////////////////////////////////////////////////////// -#ifndef __PYTHON_POINTDATA__ -#define __PYTHON_POINTDATA__ +#ifndef __PYTHON_POINTDATA_H__ +#define __PYTHON_POINTDATA_H__ //////////////////////////////////////////////////////////////////////// @@ -72,8 +72,11 @@ extern void print_met_data(MetPointObsData *obs_data, MetPointHeader *header_dat //////////////////////////////////////////////////////////////////////// +#include "python_pointdata.hpp" -#endif /* __PYTHON_POINTDATA__ */ +//////////////////////////////////////////////////////////////////////// + +#endif /* __PYTHON_POINTDATA_H__ */ //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_pointdata_python/python_pointdata.hpp b/src/libcode/vx_pointdata_python/python_pointdata.hpp new file mode 100644 index 0000000000..c4ade37053 --- /dev/null +++ b/src/libcode/vx_pointdata_python/python_pointdata.hpp @@ -0,0 +1,79 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2022 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + + +#ifndef __PYTHON_POINTDATA_HPP__ +#define __PYTHON_POINTDATA_HPP__ + +#include "vx_python3_utils.h" + +//////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////// + +template +static void set_array_from_python(PyObject *python_data, const char *python_key, T *out, bool required=true) { + const char *method_name = "set_array_from_python(T *) -> "; + PyObject *numpy_array_obj = PyDict_GetItemString (python_data, python_key); + if (numpy_array_obj) { + bool status = false; + ConcatString py_type_name = Py_TYPE(numpy_array_obj)->tp_name; + if ("numpy.ndarray" == py_type_name || "MaskedArray" == py_type_name ){ + Python3_Numpy np; + np.set(numpy_array_obj); + pointdata_from_np_array(np, out); + status = true; + } + else if (PyList_Check(numpy_array_obj)) { + status = pointdata_from_python_list(numpy_array_obj, out); + if(!status) { + if (PyFloat_Check(PyList_GetItem(numpy_array_obj, 0))) { + mlog << Error << "\n" << method_name + << "Only int type is supported at python list for " << python_key << "." + << " Check the data type from python and consider using numpy for python embedding\n\n"; + } + else { + mlog << Error << "\n" << method_name + << "Only int/float type is supported at python list for " << python_key << "." + << " Check the data type from python and consider using numpy for python embedding\n\n"; + } + } + } + else { + mlog << Error << "\n" << method_name + << "Not getting the point data by the key (" << python_key << ") from python object\n" + << " The python type \"" << py_type_name << "\" is not supported\n\n"; + } + if (status) { + mlog << Debug(7) << method_name + << "get the point data for " << python_key << " from python object\n"; + } + else exit (1); + } + else { + if (required) { + mlog << Error << "\n" << method_name + << "error getting the point data by the key (" << python_key << ") from python object\n\n"; + exit (1); + } + else mlog << Debug(3) << method_name + << "not exists the point data (" << python_key << ") from python object\n"; + } +} + + +//////////////////////////////////////////////////////////////////////// + + +#endif /* __PYTHON_POINTDATA_HPP___ */ + + +//////////////////////////////////////////////////////////////////////// +