From 65b2663f704aab5a7459ea0aedcaa8cad0ab13e7 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Tue, 16 Jan 2024 08:50:32 -0700 Subject: [PATCH] Feature #2701 ismn (#2758) * Add support for ISMN soil moisture data * Per #2701, update ascii2nc.cc with the ismn file option. * Per #2701, update ascii2nc docs * Per #2701, store the depth for precip as 0. * Per #2701, parse the ISMN observation value from the correct column. * Per #2701, every time I run ascii2nc to I see a log message for 'DEBUG 1: Number of NDBC skipped files due to no lookup 0'. This is printed by the NdbcHandler destructor. Update the logic to only print a debug level 3 log message when the number of missing locations is greater than 0, 'DEBUG 3: Skipped 5 NDBC files whose locations are not defined in 'MET_BASE/table_files/ndbc_stations.xml'. Set the MET_NDBC_STATIONS environment variable to override this file.' * Per #2701, as instructed by @anewman89, store the average of the depth values rather than the maximum value. * Per #2701, add a test of processing the ISMN data through ascii2nc. * Per #2701, fix parsing of year and month to subtract 1900 and 1, respectively. * Per #2701, set use_var_id to true for ISMN inputs * Per #2701, even though we're mapping obs data to GRIB code names and units, we want to encode them with use_var_id = true and so we need to keep track of those var_id values. * Per #2701, fix typo in the name of the ascii2nc netcdf output file. --------- Co-authored-by: MET Tools Test Account --- docs/Users_Guide/reformat_point.rst | 4 +- internal/test_unit/xml/unit_ascii2nc.xml | 13 +- src/tools/other/ascii2nc/Makefile.am | 4 +- src/tools/other/ascii2nc/Makefile.in | 28 ++- src/tools/other/ascii2nc/aeronet_handler.h | 6 +- src/tools/other/ascii2nc/airnow_handler.cc | 2 +- src/tools/other/ascii2nc/airnow_handler.h | 6 +- src/tools/other/ascii2nc/ascii2nc.cc | 27 ++- src/tools/other/ascii2nc/file_handler.h | 6 +- src/tools/other/ascii2nc/ismn_handler.cc | 255 ++++++++++++++++++++ src/tools/other/ascii2nc/ismn_handler.h | 154 ++++++++++++ src/tools/other/ascii2nc/little_r_handler.h | 6 +- src/tools/other/ascii2nc/met_handler.h | 6 +- src/tools/other/ascii2nc/ndbc_handler.cc | 13 +- src/tools/other/ascii2nc/ndbc_handler.h | 4 +- src/tools/other/ascii2nc/surfrad_handler.h | 6 +- src/tools/other/ascii2nc/wwsis_handler.h | 6 +- 17 files changed, 510 insertions(+), 36 deletions(-) create mode 100644 src/tools/other/ascii2nc/ismn_handler.cc create mode 100644 src/tools/other/ascii2nc/ismn_handler.h diff --git a/docs/Users_Guide/reformat_point.rst b/docs/Users_Guide/reformat_point.rst index d21586ebed..31d752a6f3 100644 --- a/docs/Users_Guide/reformat_point.rst +++ b/docs/Users_Guide/reformat_point.rst @@ -456,6 +456,8 @@ While initial versions of the ASCII2NC tool only supported a simple 11 column AS • `National Data Buoy (NDBC) Standard Meteorlogical Data format `_. See the :ref:`MET_NDBC_STATIONS` environment variable. +• `International Soil Moisture Network (ISMN) Data format `_. + • `AErosol RObotic NEtwork (AERONET) versions 2 and 3 format `_ • Python embedding of point observations, as described in :numref:`pyembed-point-obs-data`. See example below in :numref:`ascii2nc-pyembed`. @@ -539,7 +541,7 @@ Required Arguments for ascii2nc Optional Arguments for ascii2nc ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -3. The **-format ASCII_format** option may be set to "met_point", "little_r", "surfrad", "wwsis", "airnowhourlyaqobs", "airnowhourly", "airnowdaily_v2", "ndbc_standard", "aeronet", "aeronetv2", "aeronetv3", or "python". If passing in ISIS data, use the "surfrad" format flag. +3. The **-format ASCII_format** option may be set to "met_point", "little_r", "surfrad", "wwsis", "airnowhourlyaqobs", "airnowhourly", "airnowdaily_v2", "ndbc_standard", "ismn", "aeronet", "aeronetv2", "aeronetv3", or "python". If passing in ISIS data, use the "surfrad" format flag. 4. The **-config file** option is the configuration file for generating time summaries. diff --git a/internal/test_unit/xml/unit_ascii2nc.xml b/internal/test_unit/xml/unit_ascii2nc.xml index 77f445e1c0..2dd9df07e7 100644 --- a/internal/test_unit/xml/unit_ascii2nc.xml +++ b/internal/test_unit/xml/unit_ascii2nc.xml @@ -199,7 +199,16 @@ - - + + &MET_BIN;/ascii2nc + \ + -format ismn \ + &DATA_DIR_OBS;/ismn/SNOTEL/*/*.stm \ + &OUTPUT_DIR;/ascii2nc/ismn_SNOTEL_20220924_20220927.nc + + + &OUTPUT_DIR;/ascii2nc/ismn_SNOTEL_20220924_20220927.nc + + diff --git a/src/tools/other/ascii2nc/Makefile.am b/src/tools/other/ascii2nc/Makefile.am index ae5cf6402c..f4cef08512 100644 --- a/src/tools/other/ascii2nc/Makefile.am +++ b/src/tools/other/ascii2nc/Makefile.am @@ -28,7 +28,9 @@ ascii2nc_SOURCES = ascii2nc.cc \ ndbc_handler.cc ndbc_handler.h \ ndbc_locations.cc ndbc_locations.h \ airnow_locations.cc airnow_locations.h \ - aeronet_handler.cc aeronet_handler.h $(OPT_PYTHON_SOURCES) + aeronet_handler.cc aeronet_handler.h \ + ismn_handler.cc ismn_handler.h \ + $(OPT_PYTHON_SOURCES) ascii2nc_CPPFLAGS = ${MET_CPPFLAGS} -I../../../basic/vx_log ascii2nc_LDFLAGS = ${MET_LDFLAGS} diff --git a/src/tools/other/ascii2nc/Makefile.in b/src/tools/other/ascii2nc/Makefile.in index 00f20b1511..ea6e06e27f 100644 --- a/src/tools/other/ascii2nc/Makefile.in +++ b/src/tools/other/ascii2nc/Makefile.in @@ -110,7 +110,8 @@ am__ascii2nc_SOURCES_DIST = ascii2nc.cc ascii2nc_conf_info.cc \ airnow_handler.h ndbc_handler.cc ndbc_handler.h \ ndbc_locations.cc ndbc_locations.h airnow_locations.cc \ airnow_locations.h aeronet_handler.cc aeronet_handler.h \ - python_handler.h python_handler.cc + ismn_handler.cc ismn_handler.h python_handler.h \ + python_handler.cc @ENABLE_PYTHON_TRUE@am__objects_1 = ascii2nc-python_handler.$(OBJEXT) am__objects_2 = $(am__objects_1) am_ascii2nc_OBJECTS = ascii2nc-ascii2nc.$(OBJEXT) \ @@ -124,7 +125,8 @@ am_ascii2nc_OBJECTS = ascii2nc-ascii2nc.$(OBJEXT) \ ascii2nc-ndbc_handler.$(OBJEXT) \ ascii2nc-ndbc_locations.$(OBJEXT) \ ascii2nc-airnow_locations.$(OBJEXT) \ - ascii2nc-aeronet_handler.$(OBJEXT) $(am__objects_2) + ascii2nc-aeronet_handler.$(OBJEXT) \ + ascii2nc-ismn_handler.$(OBJEXT) $(am__objects_2) ascii2nc_OBJECTS = $(am_ascii2nc_OBJECTS) am__DEPENDENCIES_1 = ascii2nc_DEPENDENCIES = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ @@ -153,6 +155,7 @@ am__depfiles_remade = ./$(DEPDIR)/ascii2nc-aeronet_handler.Po \ ./$(DEPDIR)/ascii2nc-ascii2nc.Po \ ./$(DEPDIR)/ascii2nc-ascii2nc_conf_info.Po \ ./$(DEPDIR)/ascii2nc-file_handler.Po \ + ./$(DEPDIR)/ascii2nc-ismn_handler.Po \ ./$(DEPDIR)/ascii2nc-little_r_handler.Po \ ./$(DEPDIR)/ascii2nc-met_handler.Po \ ./$(DEPDIR)/ascii2nc-ndbc_handler.Po \ @@ -387,7 +390,9 @@ ascii2nc_SOURCES = ascii2nc.cc \ ndbc_handler.cc ndbc_handler.h \ ndbc_locations.cc ndbc_locations.h \ airnow_locations.cc airnow_locations.h \ - aeronet_handler.cc aeronet_handler.h $(OPT_PYTHON_SOURCES) + aeronet_handler.cc aeronet_handler.h \ + ismn_handler.cc ismn_handler.h \ + $(OPT_PYTHON_SOURCES) ascii2nc_CPPFLAGS = ${MET_CPPFLAGS} -I../../../basic/vx_log ascii2nc_LDFLAGS = ${MET_LDFLAGS} @@ -514,6 +519,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ascii2nc-ascii2nc.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ascii2nc-ascii2nc_conf_info.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ascii2nc-file_handler.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ascii2nc-ismn_handler.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ascii2nc-little_r_handler.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ascii2nc-met_handler.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/ascii2nc-ndbc_handler.Po@am__quote@ # am--include-marker @@ -710,6 +716,20 @@ ascii2nc-aeronet_handler.obj: aeronet_handler.cc @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ascii2nc_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o ascii2nc-aeronet_handler.obj `if test -f 'aeronet_handler.cc'; then $(CYGPATH_W) 'aeronet_handler.cc'; else $(CYGPATH_W) '$(srcdir)/aeronet_handler.cc'; fi` +ascii2nc-ismn_handler.o: ismn_handler.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ascii2nc_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT ascii2nc-ismn_handler.o -MD -MP -MF $(DEPDIR)/ascii2nc-ismn_handler.Tpo -c -o ascii2nc-ismn_handler.o `test -f 'ismn_handler.cc' || echo '$(srcdir)/'`ismn_handler.cc +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ascii2nc-ismn_handler.Tpo $(DEPDIR)/ascii2nc-ismn_handler.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ismn_handler.cc' object='ascii2nc-ismn_handler.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ascii2nc_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o ascii2nc-ismn_handler.o `test -f 'ismn_handler.cc' || echo '$(srcdir)/'`ismn_handler.cc + +ascii2nc-ismn_handler.obj: ismn_handler.cc +@am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ascii2nc_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT ascii2nc-ismn_handler.obj -MD -MP -MF $(DEPDIR)/ascii2nc-ismn_handler.Tpo -c -o ascii2nc-ismn_handler.obj `if test -f 'ismn_handler.cc'; then $(CYGPATH_W) 'ismn_handler.cc'; else $(CYGPATH_W) '$(srcdir)/ismn_handler.cc'; fi` +@am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ascii2nc-ismn_handler.Tpo $(DEPDIR)/ascii2nc-ismn_handler.Po +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='ismn_handler.cc' object='ascii2nc-ismn_handler.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ascii2nc_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -c -o ascii2nc-ismn_handler.obj `if test -f 'ismn_handler.cc'; then $(CYGPATH_W) 'ismn_handler.cc'; else $(CYGPATH_W) '$(srcdir)/ismn_handler.cc'; fi` + ascii2nc-python_handler.o: python_handler.cc @am__fastdepCXX_TRUE@ $(AM_V_CXX)$(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(ascii2nc_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) -MT ascii2nc-python_handler.o -MD -MP -MF $(DEPDIR)/ascii2nc-python_handler.Tpo -c -o ascii2nc-python_handler.o `test -f 'python_handler.cc' || echo '$(srcdir)/'`python_handler.cc @am__fastdepCXX_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/ascii2nc-python_handler.Tpo $(DEPDIR)/ascii2nc-python_handler.Po @@ -857,6 +877,7 @@ distclean: distclean-am -rm -f ./$(DEPDIR)/ascii2nc-ascii2nc.Po -rm -f ./$(DEPDIR)/ascii2nc-ascii2nc_conf_info.Po -rm -f ./$(DEPDIR)/ascii2nc-file_handler.Po + -rm -f ./$(DEPDIR)/ascii2nc-ismn_handler.Po -rm -f ./$(DEPDIR)/ascii2nc-little_r_handler.Po -rm -f ./$(DEPDIR)/ascii2nc-met_handler.Po -rm -f ./$(DEPDIR)/ascii2nc-ndbc_handler.Po @@ -915,6 +936,7 @@ maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/ascii2nc-ascii2nc.Po -rm -f ./$(DEPDIR)/ascii2nc-ascii2nc_conf_info.Po -rm -f ./$(DEPDIR)/ascii2nc-file_handler.Po + -rm -f ./$(DEPDIR)/ascii2nc-ismn_handler.Po -rm -f ./$(DEPDIR)/ascii2nc-little_r_handler.Po -rm -f ./$(DEPDIR)/ascii2nc-met_handler.Po -rm -f ./$(DEPDIR)/ascii2nc-ndbc_handler.Po diff --git a/src/tools/other/ascii2nc/aeronet_handler.h b/src/tools/other/ascii2nc/aeronet_handler.h index ec64202944..e835a91602 100644 --- a/src/tools/other/ascii2nc/aeronet_handler.h +++ b/src/tools/other/ascii2nc/aeronet_handler.h @@ -10,8 +10,8 @@ //////////////////////////////////////////////////////////////////////// -#ifndef __AERONETHANDLER_H__ -#define __AERONETHANDLER_H__ +#ifndef __AERONET_HANDLER_H__ +#define __AERONET_HANDLER_H__ //////////////////////////////////////////////////////////////////////// @@ -125,7 +125,7 @@ class AeronetHandler : public FileHandler //////////////////////////////////////////////////////////////////////// -#endif /* __AERONETHANDLER_H__ */ +#endif /* __AERONET_HANDLER_H__ */ //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/ascii2nc/airnow_handler.cc b/src/tools/other/ascii2nc/airnow_handler.cc index a5542b642c..74525c6139 100644 --- a/src/tools/other/ascii2nc/airnow_handler.cc +++ b/src/tools/other/ascii2nc/airnow_handler.cc @@ -648,7 +648,7 @@ time_t AirnowHandler::_getValidTime(const string &dateStr, const string &timeStr struct tm time_struct; memset(&time_struct, 0, sizeof(time_struct)); - time_struct.tm_year = atoi(year.c_str()) -1900; + time_struct.tm_year = atoi(year.c_str()) - 1900; time_struct.tm_mon = atoi(mon.c_str()) - 1; time_struct.tm_mday = atoi(mday.c_str()); time_struct.tm_hour = atoi(hour.c_str()); diff --git a/src/tools/other/ascii2nc/airnow_handler.h b/src/tools/other/ascii2nc/airnow_handler.h index 9064c5ce70..beef7bf0d5 100644 --- a/src/tools/other/ascii2nc/airnow_handler.h +++ b/src/tools/other/ascii2nc/airnow_handler.h @@ -10,8 +10,8 @@ //////////////////////////////////////////////////////////////////////// -#ifndef __AIRNOWHANDLER_H__ -#define __AIRNOWHANDLER_H__ +#ifndef __AIRNOW_HANDLER_H__ +#define __AIRNOW_HANDLER_H__ //////////////////////////////////////////////////////////////////////// @@ -198,7 +198,7 @@ class AirnowHandler : public FileHandler //////////////////////////////////////////////////////////////////////// -#endif /* __AERONETHANDLER_H__ */ +#endif /* __AERONET_HANDLER_H__ */ //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/ascii2nc/ascii2nc.cc b/src/tools/other/ascii2nc/ascii2nc.cc index 19cfa25549..9caecd0481 100644 --- a/src/tools/other/ascii2nc/ascii2nc.cc +++ b/src/tools/other/ascii2nc/ascii2nc.cc @@ -48,7 +48,8 @@ // 019 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main // 020 08/26/22 Dave Albo MET #2142 Add AirNow observations // 021 10/03/22 Prestopnik MET #2227 Remove using namespace std from header files -// 022 10/07/22 Dave Albo MET #2276 Add NDBC Buoy data +// 022 10/07/22 Dave Albo MET #2276 Add NDBC buoy data +// 023 11/28/23 Halley Gotway MET #2701 Add ISMN soil moisture data // //////////////////////////////////////////////////////////////////////// @@ -86,6 +87,7 @@ using namespace std; #include "aeronet_handler.h" #include "airnow_handler.h" #include "ndbc_handler.h" +#include "ismn_handler.h" #ifdef ENABLE_PYTHON #include "global_python.h" @@ -113,6 +115,7 @@ enum ASCIIFormat { ASCIIFormat_Airnow_hourlyaqobs, ASCIIFormat_Airnow_hourly, ASCIIFormat_NDBC_standard, + ASCIIFormat_ISMN, ASCIIFormat_Aeronet_v2, ASCIIFormat_Aeronet_v3, ASCIIFormat_Python, @@ -322,6 +325,10 @@ FileHandler *create_file_handler(const ASCIIFormat format, const ConcatString &a return((FileHandler *) handler); } + case ASCIIFormat_ISMN: { + return((FileHandler *) new IsmnHandler(program_name)); + } + case ASCIIFormat_Aeronet_v2: { AeronetHandler *handler = new AeronetHandler(program_name); handler->setFormatVersion(2); @@ -459,9 +466,21 @@ FileHandler *determine_ascii_format(const ConcatString &ascii_filename) { delete ndbc_file; // - // If we get here, we didn't recognize the file contents. + // See if this is an ISMN file. // + f_in.rewind(); + IsmnHandler *ismn_file = new IsmnHandler(program_name); + + if(ismn_file->isFileType(f_in)) { + f_in.close(); + return((FileHandler *) ismn_file); + } + + delete ismn_file; + // + // If we get here, we didn't recognize the file contents. + // mlog << Error << "\ndetermine_ascii_format() -> " << "could not determine file format based on file contents\n\n"; @@ -503,6 +522,7 @@ void usage() { << AirnowHandler::getFormatStringHourlyAqObs() << "\", \"" << AirnowHandler::getFormatStringHourly() << "\", \"" << NdbcHandler::getFormatStringStandard() << "\", \"" + << IsmnHandler::getFormatString() << "\", \"" << AeronetHandler::getFormatString() << "\", \"" << AeronetHandler::getFormatString_v2() << "\", \"" << AeronetHandler::getFormatString_v3() << "\""; @@ -585,6 +605,9 @@ void set_format(const StringArray & a) { else if(NdbcHandler::getFormatStringStandard() == a[0]) { ascii_format = ASCIIFormat_NDBC_standard; } + else if(IsmnHandler::getFormatString() == a[0]) { + ascii_format = ASCIIFormat_ISMN; + } else if(AeronetHandler::getFormatString() == a[0] || AeronetHandler::getFormatString_v2() == a[0]) { ascii_format = ASCIIFormat_Aeronet_v2; diff --git a/src/tools/other/ascii2nc/file_handler.h b/src/tools/other/ascii2nc/file_handler.h index ece575672a..8358411f36 100644 --- a/src/tools/other/ascii2nc/file_handler.h +++ b/src/tools/other/ascii2nc/file_handler.h @@ -10,8 +10,8 @@ //////////////////////////////////////////////////////////////////////// -#ifndef __FILEHANDLER_H__ -#define __FILEHANDLER_H__ +#ifndef __FILE_HANDLER_H__ +#define __FILE_HANDLER_H__ //////////////////////////////////////////////////////////////////////// @@ -158,7 +158,7 @@ inline void FileHandler::setMessageTypeMap(map m) { //////////////////////////////////////////////////////////////////////// -#endif /* __FILEHANDLER_H__ */ +#endif /* __FILE_HANDLER_H__ */ //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/ascii2nc/ismn_handler.cc b/src/tools/other/ascii2nc/ismn_handler.cc new file mode 100644 index 0000000000..0cf8c29042 --- /dev/null +++ b/src/tools/other/ascii2nc/ismn_handler.cc @@ -0,0 +1,255 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2023 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +using namespace std; + +#include + +#include "vx_log.h" +#include "vx_math.h" +#include "vx_util.h" + +#include "ismn_handler.h" + +const int IsmnHandler::MIN_NUM_HDR_COLS = 8; +const int IsmnHandler::NUM_OBS_COLS = 5; + +// Relevant GRIB codes +static const int PRATE_GRIB_CODE = 59; +static const int SNOD_GRIB_CODE = 66; +static const int SOILW_GRIB_CODE = 144; +static const int SMS_GRIB_CODE = -1; +static const int WEASD_GRIB_CODE = 65; +static const int TMP_GRIB_CODE = 11; +static const int TSOIL_GRIB_CODE = 85; +static const int AVSFT_GRIB_CODE = 148; + +// Mapping of ISMN strings to output variable names +map IsmnObsVarMap = { + { "p", { PRATE_GRIB_CODE, "PRATE" } }, + { "sd", { SNOD_GRIB_CODE, "SNOD" } }, + { "sm", { SOILW_GRIB_CODE, "SOILW" } }, + { "su", { SMS_GRIB_CODE, "SMS" } }, + { "sweq", { WEASD_GRIB_CODE, "WEASD" } }, + { "ta", { TMP_GRIB_CODE, "TMP" } }, + { "ts", { TSOIL_GRIB_CODE, "TSOIL" } }, + { "tsf", { AVSFT_GRIB_CODE, "AVSFT" } } +}; + +//////////////////////////////////////////////////////////////////////// +// +// Code for class IsmnHandler +// +//////////////////////////////////////////////////////////////////////// + +IsmnHandler::IsmnHandler(const string &program_name) : + FileHandler(program_name) { + use_var_id = true; +} + +//////////////////////////////////////////////////////////////////////// + +IsmnHandler::~IsmnHandler() { } + +//////////////////////////////////////////////////////////////////////// + +bool IsmnHandler::isFileType(LineDataFile &ascii_file) const { + + // ISMN files are identified by having a .stm suffix and + // checking the number of header and data columns. + // The header and data lines look like this: + // MAQU MAQU NST_24 33.99908 102.13661 3449.0 0.4000 0.4000 ECH20 EC-TM + // 2014/10/21 07:00 -27.7 G M + + // Initialize using the filename suffix + bool is_file_type = check_prefix_suffix(ascii_file.short_filename(), + nullptr, ".stm"); + + // Read the header line + DataLine dl; + while(dl.n_items() == 0) ascii_file >> dl; + + // Check the minimum number of header columns + if(dl.n_items() < MIN_NUM_HDR_COLS) is_file_type = false; + + // Check the number of data line columns + ascii_file >> dl; + if(dl.n_items() != NUM_OBS_COLS) is_file_type = false; + + return(is_file_type); +} + +//////////////////////////////////////////////////////////////////////// +// Private/Protected methods +//////////////////////////////////////////////////////////////////////// + +bool IsmnHandler::_readObservations(LineDataFile &ascii_file) { + + // Read and save the header information + if(!_readHeaderInfo(ascii_file)) return(false); + + // Get the var_id to use + int var_id = bad_data_int; + if(!_varNames.has(_obsVarInfo._varName, var_id)) return(false); + + // Process the observation lines + DataLine dl; + while(ascii_file >> dl) { + + // Make sure that the line contains the correct number of tokens + if(dl.n_items() != NUM_OBS_COLS) { + mlog << Error << "\nIsmnHandler::_readObservations() -> " + << "unexpected number of columns (" << dl.n_items() + << " != " << NUM_OBS_COLS << ") on line number " + << dl.line_number() << " of ISMN file \"" + << ascii_file.filename() << "\"!\n\n"; + return(false); + } + + // Extract the valid time from the data line + time_t valid_time = _getValidTime(dl); + if(valid_time == 0) return(false); + + // Store the observation value + double obs_value = atof(dl[2]); + + // Handle unit conversion + switch(_obsVarInfo._gribCode) { + + // Convert precip rate in second to hours + case PRATE_GRIB_CODE: + obs_value /= 3600; + break; + + // Convert mm to m + case SNOD_GRIB_CODE: + obs_value /= 1000.0; + break; + + // Convert C to K + case TMP_GRIB_CODE: + case TSOIL_GRIB_CODE: + case AVSFT_GRIB_CODE: + obs_value += 273.15; + break; + + default: + break; + } + + // Store the observation + _addObservations(Observation( + _networkName, _stationId, valid_time, + _stationLat, _stationLon, _stationElv, + dl[3], var_id, bad_data_double, + _depth, obs_value, _obsVarInfo._varName)); + + } // end while + + return(true); +} + +//////////////////////////////////////////////////////////////////////// + +time_t IsmnHandler::_getValidTime(const DataLine &dl) const { + struct tm time_struct; + memset(&time_struct, 0, sizeof(time_struct)); + + // Formatted as YYYY/MM/DD HH:MM + string ymd_str(dl[0]); + string hm_str(dl[1]); + + // Validate the time strings + if(!check_reg_exp("^[0-9]\\{4\\}/[0-9]\\{2\\}/[0-9]\\{2\\}$", dl[0]) || + !check_reg_exp("^[0-9]\\{2\\}:[0-9]\\{2\\}$", dl[1])) { + mlog << Warning << "\nIsmnHandler::_getValidTime() -> " + << "unexpected time stamp format on line number " + << dl.line_number() << " of ISMN input file:\n" + << " " << dl << "\n\n"; + } + else { + + // Parse time components + time_struct.tm_year = stoi(ymd_str.substr(0, 4)) - 1900; + time_struct.tm_mon = stoi(ymd_str.substr(5, 2)) - 1; + time_struct.tm_mday = stoi(ymd_str.substr(8, 2)); + time_struct.tm_hour = stoi( hm_str.substr(0, 2)); + time_struct.tm_min = stoi( hm_str.substr(3, 2)); + } + + return(timegm(&time_struct)); +} + +//////////////////////////////////////////////////////////////////////// + +bool IsmnHandler::_readHeaderInfo(LineDataFile &ascii_file) { + + // The file name is delimited with underscores and the variable name + // is the fourth item + ConcatString cs(ascii_file.short_filename()); + StringArray sa = cs.split("_"); + + // Validate the file name + if(sa.n() < 4) { + mlog << Error << "\nIsmnHandler::_readHeaderInfo() -> " + << "unexpected ISMN file name \"" << ascii_file.filename() + << "\"!\n\n"; + return(false); + } + + // Validate the variable name + if(IsmnObsVarMap.count(sa[3]) == 0) { + mlog << Error << "\nIsmnHandler::_readHeaderInfo() -> " + << "unexpected variable name (" << sa[3] + << ") found in ISMN file name \"" << ascii_file.filename() + << "\"!\n\n"; + return(false); + } + + // Store the observation variable info + _obsVarInfo = IsmnObsVarMap[sa[3]]; + + // Store the output variable name + _varNames.add_uniq(_obsVarInfo._varName); + + // Read the header line + DataLine dl; + while(dl.n_items() == 0) ascii_file >> dl; + + // Check the minimum number of header columns + if(dl.n_items() < MIN_NUM_HDR_COLS) { + mlog << Error << "\nIsmnHandler::_readHeaderInfo() -> " + << "unexpected number of header columns (" + << dl.n_items() << " < " << MIN_NUM_HDR_COLS + << ") in ISMN file \"" << ascii_file.filename() + << "\"!\n\n"; + return(false); + } + + // Store the header information + _networkName = dl[1]; + _stationId = dl[2]; + _stationLat = atof(dl[3]); + _stationLon = atof(dl[4]); + _stationElv = atof(dl[5]); + + // Set the depth for precip as 0 + if(_obsVarInfo._gribCode == PRATE_GRIB_CODE) { + _depth = 0.0; + } + // Otherwise, store the average of the two depths + else { + _depth = (atof(dl[6]) + atof(dl[7]))/2.0; + } + + return(true); +} + +//////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/ascii2nc/ismn_handler.h b/src/tools/other/ascii2nc/ismn_handler.h new file mode 100644 index 0000000000..05470aedec --- /dev/null +++ b/src/tools/other/ascii2nc/ismn_handler.h @@ -0,0 +1,154 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2023 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +#ifndef __ISMN_HANDLER_H__ +#define __ISMN_HANDLER_H__ + +//////////////////////////////////////////////////////////////////////// + +#include +#include + +#include "file_handler.h" + +//////////////////////////////////////////////////////////////////////// + +struct obsVarInfo { + int _gribCode; + string _varName; +}; + +//////////////////////////////////////////////////////////////////////// +// +// International Soil Moisture Network Data +// https://ismn.bafg.de/en/data/header-value-files +// +// Dataset Filename Convention: +// CSE_Network_Station_Variablename_depthfrom_depthto_sensorname_startdate_enddate.ext +// +// - CSE: Continental Scale Experiment (CSE) acronym, if not applicable use Networkname +// - Network: Network abbreviation (e.g., OZNET) +// - Station: Station name (e.g., Widgiewa) +// - Variablename: Name of the variable in the file (e.g., Soil-Moisture) +// - p: preciptation in mm/h +// - Store as GRIB Code 59 (PRATE in kg/m^2/s) +// - Convert from mm/h to kg/m^2/s +// - sd: snow depth in mm +// - Store as GRIB Code 66 (SNOD in m) +// - Convert from mm to m +// - sm: soil moisture in kg^3/kg^3 +// - Store as GRIB Code 144 (SOILW as a fraction) +// - su: soil suction in kPa +// - Store as GRIB Code -1 (undefined, SMS in kPa) +// - sweq: snow water equivalent in mm +// - Store as GRIB Code 65 (WEASD kg/m^2) +// - ta: air temperature in C +// - Store as GRIB Code 11 (TMP in K) +// - Convert from C to K +// - ts: soil temperature in C +// - Store as GRIB Code 85 (TSOIL in K) +// - Convert from C to K +// - tsf: surface temperature in C +// - Store as GRIB Code 148 (AVSFT in K) +// - Convert from C to K +// - depthfrom: Depth in the ground in which the variable was observed (upper boundary) +// - depthto: Depth in the ground in which the variable was observed (lower boundary) +// - sensorname: Name of the sensor used +// - startdate: Date of the first dataset in the file (format YYYYMMDD) +// - enddate: Date of the last dataset in the file (format YYYYMMDD) +// - ext: Extension .stm (Soil Temperature and Soil Moisture Data Set see CEOP standard) +// +// Example: OZNET_OZNET_Widgiewa_Soil-Temperature_0.150000_0.150000_20010103_20090812.stm +// +// Dataset Conents: +// Example: +// REMEDHUS REMEDHUS Zamarron 41.24100 -5.54300 855.00 0.05 0.05 +// 2005/03/16 00:00 10.30 U M +// 2005/03/16 01:00 9.80 U M +// ... +// +// Header Line: +// CSE, Network, Station, +// Latitude (degrees north), Longitude (degrees east), +// Elevation (msl), Depth from, Depth to, +// Sensor name (Note: contains embedded whitespace) +// +// Record Lines: +// YYYY/MM/DD HH:MM, Variable value, +// ISMN Quality Flag, Data Provider Quality Flag +// +//////////////////////////////////////////////////////////////////////// + +class IsmnHandler : public FileHandler { + + public: + + IsmnHandler(const string &program_name); + virtual ~IsmnHandler(); + + virtual bool isFileType(LineDataFile &ascii_file) const; + + static string getFormatString() { return "ismn"; } + + protected: + + ///////////////////////// + // Protected constants + ///////////////////////// + + // The number of columns in the second header line in the file. This line + // is used to determine if this is a ISMN file since the first line has + // an indeterminate number of tokens. + + static const int MIN_NUM_HDR_COLS; + + // The number of columns in the observation lines in the file. + + static const int NUM_OBS_COLS; + + /////////////////////// + // Protected members + /////////////////////// + + // Unchanging file name information + obsVarInfo _obsVarInfo; + + // Store list of unqiue output variable names + StringArray _varNames; + + // Unchanging header information + string _networkName; + string _stationId; + double _stationLat; + double _stationLon; + double _stationElv; + double _depth; + + /////////////////////// + // Protected methods + /////////////////////// + + // Read and save the header information from the given file + bool _readHeaderInfo(LineDataFile &ascii_file); + + // Get the valid time from the observation line + time_t _getValidTime(const DataLine &data_line) const; + + // Read the observations and add them to the + // _observations vector + virtual bool _readObservations(LineDataFile &ascii_file); + +}; + +//////////////////////////////////////////////////////////////////////// + +#endif /* __ISMN_HANDLER_H__ */ + +//////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/ascii2nc/little_r_handler.h b/src/tools/other/ascii2nc/little_r_handler.h index 897f2416c3..c810a45ee4 100644 --- a/src/tools/other/ascii2nc/little_r_handler.h +++ b/src/tools/other/ascii2nc/little_r_handler.h @@ -10,8 +10,8 @@ //////////////////////////////////////////////////////////////////////// -#ifndef __LITTLERHANDLER_H__ -#define __LITTLERHANDLER_H__ +#ifndef __LITTLER_HANDLER_H__ +#define __LITTLER_HANDLER_H__ //////////////////////////////////////////////////////////////////////// @@ -64,7 +64,7 @@ class LittleRHandler : public FileHandler //////////////////////////////////////////////////////////////////////// -#endif /* __LITTLERHANDLER_H__ */ +#endif /* __LITTLER_HANDLER_H__ */ //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/ascii2nc/met_handler.h b/src/tools/other/ascii2nc/met_handler.h index f8dd880508..51ecf08010 100644 --- a/src/tools/other/ascii2nc/met_handler.h +++ b/src/tools/other/ascii2nc/met_handler.h @@ -10,8 +10,8 @@ //////////////////////////////////////////////////////////////////////// -#ifndef __METHANDLER_H__ -#define __METHANDLER_H__ +#ifndef __MET_HANDLER_H__ +#define __MET_HANDLER_H__ //////////////////////////////////////////////////////////////////////// @@ -54,7 +54,7 @@ class MetHandler : public FileHandler //////////////////////////////////////////////////////////////////////// -#endif /* __METHANDLER_H__ */ +#endif /* __MET_HANDLER_H__ */ //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/ascii2nc/ndbc_handler.cc b/src/tools/other/ascii2nc/ndbc_handler.cc index 9ec04255a2..510984f541 100644 --- a/src/tools/other/ascii2nc/ndbc_handler.cc +++ b/src/tools/other/ascii2nc/ndbc_handler.cc @@ -120,8 +120,15 @@ NdbcHandler::NdbcHandler(const string &program_name) : NdbcHandler::~NdbcHandler() { - mlog << Debug(1) << "Number of NDBC skipped files due to no lookup " << numMissingStations - << "\n"; + + // Log the non-zero number of missing stations + if(numMissingStations > 0) { + mlog << Debug(3) << "Skipped " << numMissingStations + << " NDBC files whose locations are not defined in \"" + << locationsFileName << "\". Set the " << stations_env + << " environment variable to provide an updated " + << "locations file.\n"; + } } @@ -362,7 +369,7 @@ time_t NdbcHandler::_getValidTime(const DataLine &data_line) const struct tm time_struct; memset(&time_struct, 0, sizeof(time_struct)); - time_struct.tm_year = atoi(year.c_str()) -1900; + time_struct.tm_year = atoi(year.c_str()) - 1900; time_struct.tm_mon = atoi(month.c_str()) - 1; time_struct.tm_mday = atoi(day.c_str()); time_struct.tm_hour = atoi(hour.c_str()); diff --git a/src/tools/other/ascii2nc/ndbc_handler.h b/src/tools/other/ascii2nc/ndbc_handler.h index b69b0ca311..d75b2136e5 100644 --- a/src/tools/other/ascii2nc/ndbc_handler.h +++ b/src/tools/other/ascii2nc/ndbc_handler.h @@ -10,8 +10,8 @@ //////////////////////////////////////////////////////////////////////// -#ifndef __NDBCHANDLER_H__ -#define __NDBCHANDLER_H__ +#ifndef __NDBC_HANDLER_H__ +#define __NDBC_HANDLER_H__ //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/ascii2nc/surfrad_handler.h b/src/tools/other/ascii2nc/surfrad_handler.h index 4fe8a3a9bb..ecbe6306e9 100644 --- a/src/tools/other/ascii2nc/surfrad_handler.h +++ b/src/tools/other/ascii2nc/surfrad_handler.h @@ -10,8 +10,8 @@ //////////////////////////////////////////////////////////////////////// -#ifndef __SURFRADHANDLER_H__ -#define __SURFRADHANDLER_H__ +#ifndef __SURFRAD_HANDLER_H__ +#define __SURFRAD_HANDLER_H__ //////////////////////////////////////////////////////////////////////// @@ -136,7 +136,7 @@ class SurfradHandler : public FileHandler //////////////////////////////////////////////////////////////////////// -#endif /* __SURFRADHANDLER_H__ */ +#endif /* __SURFRAD_HANDLER_H__ */ //////////////////////////////////////////////////////////////////////// diff --git a/src/tools/other/ascii2nc/wwsis_handler.h b/src/tools/other/ascii2nc/wwsis_handler.h index 52d4f8c48b..774dd3ade1 100644 --- a/src/tools/other/ascii2nc/wwsis_handler.h +++ b/src/tools/other/ascii2nc/wwsis_handler.h @@ -12,8 +12,8 @@ //////////////////////////////////////////////////////////////////////// -#ifndef __WWSISHANDLER_H__ -#define __WWSISHANDLER_H__ +#ifndef __WWSIS_HANDLER_H__ +#define __WWSIS_HANDLER_H__ //////////////////////////////////////////////////////////////////////// @@ -115,7 +115,7 @@ class WwsisHandler : public FileHandler //////////////////////////////////////////////////////////////////////// -#endif /* __WWSISHANDLER_H__ */ +#endif /* __WWSIS_HANDLER_H__ */ ////////////////////////////////////////////////////////////////////////