Update develop-ref after #2439 (#2446)

Co-authored-by: Dave Albo <dave@seneca.rap.ucar.edu> Co-authored-by: John Halley Gotway <johnhg@ucar.edu> Co-authored-by: Seth Linden <linden@seneca.rap.ucar.edu> Co-authored-by: johnhg <johnhg@ucar.edu> Co-authored-by: Lisa Goodrich <lisag@seneca.rap.ucar.edu> Co-authored-by: jprestop <jpresto@ucar.edu> Co-authored-by: Howard Soh <hsoh@seneca.rap.ucar.edu> Co-authored-by: MET Tools Test Account <met_test@seneca.rap.ucar.edu> Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Julie Prestopnik <jpresto@seneca.rap.ucar.edu> Co-authored-by: Jonathan Vigh <jvigh@ucar.edu> Co-authored-by: davidalbo <dave@ucar.edu> Co-authored-by: hsoh-u <hsoh@ucar.edu> Co-authored-by: Julie Prestopnik <jpresto@ucar.edu> Co-authored-by: Seth Linden <linden@ucar.edu> Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: Daniel Adriaansen <dadriaan@ucar.edu> fix 2238 link error (#2255) fix #2271 develop nbrctc (#2272) fix #2309 develop tcmpr (#2310) fix #2306 ascii2nc airnow hourly (#2314) fix_spread_md (#2335) fix #2389 develop flowchart (#2392) Fix Python environment issue (#2407) fix definitions of G172 and G220 based on comments in NOAA-EMC/NCEPLIBS-w3emc#157. (#2406) fix #2380 develop override (#2382) fix #2408 develop empty config (#2410) fix #2390 develop compile zlib (#2404) fix #2412 develop climo (#2422) fix #2437 develop convert (#2439) fix for develop, for #2437, forgot one reference to the search_parent for a dictionary lookup.
dtcenter · Feb 16, 2023 · 260a9ae · 260a9ae
1 parent 9c59add
commit 260a9ae
Show file tree

Hide file tree

Showing 23 changed files with 290 additions and 134 deletions.
diff --git a/.github/jobs/build_docker_image.sh b/.github/jobs/build_docker_image.sh
@@ -15,8 +15,5 @@ time_command docker build -t ${DOCKERHUB_TAG} \
     -f $DOCKERFILE_PATH ${GITHUB_WORKSPACE}
 if [ $? != 0 ]; then
   cat ${GITHUB_WORKSPACE}/docker_build.log
-  # Append the full make_install.log file
-  echo "Appending make_install.log to docker_build.log. See the logs artifact for details."
-  cat ${GITHUB_WORKSPACE}/make_install.log >> ${GITHUB_WORKSPACE}/docker_build.log
   exit 1
 fi
diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh
@@ -6,7 +6,7 @@ run_unit_tests=false
 run_diff=false
 run_update_truth=false
 met_base_repo=met-base
-met_base_tag=v1.1
+met_base_tag=v2.0_debian10
 input_data_version=develop
 truth_data_version=develop
 

diff --git a/.github/jobs/test_env_vars.sh b/.github/jobs/test_env_vars.sh
@@ -10,4 +10,4 @@ export MET_TEST_TRUTH=/data/output/met_test_truth
 export MET_TEST_DIFF=/data/output/met_test_diff
 
 export MET_TEST_RSCRIPT=/usr/bin/Rscript
-export MET_TEST_MET_PYTHON_EXE=/usr/bin/python3
+export MET_TEST_MET_PYTHON_EXE=/usr/local/bin/python3
diff --git a/.github/workflows/build_docker_and_trigger_metplus.yml b/.github/workflows/build_docker_and_trigger_metplus.yml
@@ -29,7 +29,7 @@ jobs:
         env:
           SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}-lite
           MET_BASE_REPO: met-base
-          MET_BASE_TAG: v1.1
+          MET_BASE_TAG: v2.0_debian10
 
       - name: Push Docker Image
         run: .github/jobs/push_docker_image.sh

diff --git a/data/config/GridStatConfig_default b/data/config/GridStatConfig_default
@@ -90,7 +90,6 @@ climo_mean = {
    }
 
    time_interp_method = DW_MEAN;
-   match_month        = TRUE;
    day_interval       = 31;
    hour_interval      = 6;
 }

diff --git a/docs/Users_Guide/config_options.rst b/docs/Users_Guide/config_options.rst
@@ -693,14 +693,16 @@ using the following entries:
   smoothing. The default is 120. Ignored if not Gaussian method.
 
 * The "gaussian_dx" and "gaussian_radius" settings must be in the same
-  units, such as kilometers or degress.  Their ratio
+  units, such as kilometers or degress. Their ratio
   (sigma = gaussian_radius / gaussian_dx) determines the Guassian weighting
   function.
 
 * The "convert", "censor_thresh", and "censor_val" entries are described
-  below.  When specified, these operations are applied to the output of the
-  regridding step.  The conversion operation is applied first, followed by
-  the censoring operation.
+  below. When specified, these operations are applied to the output of the
+  regridding step. The conversion operation is applied first, followed by
+  the censoring operation. Note that these operations are limited in scope.
+  They are only applied if defined within the regrid dictionary itself.
+  Settings defined at higher levels of config file context are not applied. 
 
 .. code-block:: none
 		
@@ -1318,9 +1320,8 @@ of several entires defining the climatology file names and fields to be used.
 
 * The "hour_interval" entry is an integer specifying the spacing in hours of
   the climatology data for each day. This should be set between 0 and 24,
-  with 6 and 12 being common choices. For example, use 6 for climatology data
-  with 4 times per day, such as 00Z, 06Z, 12Z, and 18Z. Use "NA" if the timing
-  of the climatology data should not be checked.
+  with 6 and 12 being common choices. Use "NA" if the timing of the
+  climatology data should not be checked.
 
 * The "day_interval" and "hour_interval" entries replace the deprecated
   entries "match_month", "match_day", and "time_step".
@@ -1729,12 +1730,12 @@ This dictionary may include the following entries:
 * The "shape" entry may be set to SQUARE or CIRCLE to specify the shape
   of the smoothing area.
 
-* The "type" entry is an array of dictionaries, each specifying an
-  interpolation method. Interpolation is performed over a N by N box
-  centered on each point, where N is the width specified. Each of these
+* The "type" entry is an array of dictionaries, each specifying one or more
+  interpolation methods and widths. Interpolation is performed over an N by N
+  box centered on each point, where N is the width specified. Each of these
   dictionaries must include:
 
-  * The "width" entry is an integer which specifies the size of the
+  * The "width" entry is an array of integers to specify the size of the
     interpolation area. The area is either a square or circle containing
     the observation point. The width value specifies the width of the
     square or diameter of the circle. A width value of 1 is interpreted
@@ -1747,7 +1748,7 @@ This dictionary may include the following entries:
     grid point closest to the observation point. For grid-to-grid
     comparisons (i.e. Grid-Stat), the width must be odd.
 
-  * The "method" entry specifies the interpolation procedure to be
+  * The "method" entry is an array of interpolation procedures to be
     applied to the points in the box:
 
     * MIN         for the minimum value
@@ -1792,6 +1793,9 @@ This dictionary may include the following entries:
     only valid smoothing methods are MIN, MAX, MEDIAN, UW_MEAN, and
     GAUSSIAN, and MAXGAUSS.
 
+  * If multiple "method" and "width" options are specified, all possible
+    permutations of their values are applied.
+
 .. code-block:: none
 		
   interp = {
@@ -1801,8 +1805,8 @@ This dictionary may include the following entries:
   
      type = [
         {
-           method = UW_MEAN;
-           width  = 1;
+           method = [ NEAREST ];
+           width  = [ 1 ];
         }
      ];
   }

diff --git a/internal/scripts/docker/build_met_docker.sh b/internal/scripts/docker/build_met_docker.sh
@@ -2,33 +2,37 @@
 
 echo "Running script to build MET in Docker"
 
-LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_configure.log
-
 source internal/scripts/environment/development.docker
 
+mkdir -p /met/logs
+
+LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_configure.log
 echo "Configuring MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}"
 ./bootstrap
-./configure --enable-grib2 --enable-mode_graphics --enable-modis --enable-lidar2nc --enable-python > ${LOG_FILE}
+./configure --enable-grib2 --enable-mode_graphics --enable-modis --enable-lidar2nc --enable-python CPPFLAGS="-I/usr/local/include -I/usr/local/include/freetype2 -I/usr/local/include/cairo" > ${LOG_FILE} 2>&1
 if [ $? != 0 ]; then
-    exit 1
+  cat ${LOG_FILE}
+  exit 1
 fi
 
 if [ ! -z "${MAKE_ARGS}" ]; then
   echo Adding make arguments: ${MAKE_ARGS}
 fi
 
-LOG_FILE=/met/MET-${MET_GIT_NAME}/make_install.log
+LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_make_install.log
 echo "Compiling MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}"
 make ${MAKE_ARGS} install > ${LOG_FILE}
 if [ $? != 0 ]; then
-    exit 1
+  cat ${LOG_FILE}
+  exit 1
 fi
 
 LOG_FILE=/met/logs/MET-${MET_GIT_NAME}_make_test.log
 echo "Testing MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}"
 make ${MAKE_ARGS} test > ${LOG_FILE} 2>&1
 if [ $? != 0 ]; then
-    exit 1
+  cat ${LOG_FILE}
+  exit 1
 fi
 
 if [[ $MET_GIT_NAME == "v"* ]]; then

diff --git a/internal/scripts/environment/development.docker b/internal/scripts/environment/development.docker
@@ -1,39 +1,22 @@
-# Define the development environment for NCAR project machine seneca
-# Based on settings in /usr/local/src/met/README.snat
+# Define the development environment for Docker
 
 # Top-level MET project directory
 MET_PROJ_DIR=`ls -1d /met/MET*`
 
 # Variables required to build MET
 export MET_DEVELOPMENT=true
 
-export MET_NETCDF=/usr/local
-export MET_HDF5INC=/usr/include
-export MET_HDF5LIB=/usr/lib64
-export MET_HDF=/usr/local/hdf
-export MET_HDFEOS=/usr/local/hdfeos
-export MET_BUFR=/usr/local
-export MET_GRIB2CINC=/usr/include
-export MET_GRIB2CLIB=/usr/lib64
-export MET_GSLINC=/usr/include/gsl
-export MET_GSLLIB=/usr/lib64
-export MET_CAIROINC=/usr/include/cairo
-export MET_CAIROLIB=/usr/lib64 
-export MET_FREETYPEINC=/usr/include/freetype2
-export MET_FREETYPELIB=/usr/lib64
-export MET_JASPERLIB=/usr/lib64
-
-export MET_PYTHON_BIN_EXE=/usr/bin/python3
-export MET_PYTHON_CC="-I/usr/include/python3.6m"
-export MET_PYTHON_LD="-L/usr/lib64 -lpython3.6m -lpthread -ldl -lutil -lm"
-
-# -D__64BIT__ is required because we've compiled libgrib2c.a with that flag
-export CFLAGS="-DUNDERSCORE -fPIC -D__64BIT__ -g"
+export MET_PYTHON=/usr/local
+export MET_PYTHON_BIN_EXE=${MET_PYTHON}/bin/python3
+export MET_PYTHON_CC="-I${MET_PYTHON}/include/python3.10"
+export MET_PYTHON_LD="-L${MET_PYTHON}/lib -lpython3.10 -lcrypt -lpthread -ldl -lutil -lm"
+
+export CFLAGS="-DUNDERSCORE -fPIC -g"
 export CXXFLAGS=${CFLAGS}
 
 # Set LDFLAGS to include -rpath settings when compiling MET
 export LDFLAGS="-Wl,--disable-new-dtags"
-export LDFLAGS="${LDFLAGS} -Wl,-rpath,/usr/local/lib:/usr/lib64:${MET_HDF}/lib:${MET_HDFEOS}/lib"
+export LDFLAGS="${LDFLAGS} -Wl,-rpath,/usr/local/lib:/usr/lib64"
 
 # Variables required to run MET
 export MET_TEST_INPUT=${MET_PROJ_DIR}/MET_test_data/unit_test
@@ -44,3 +27,16 @@ export MET_TEST_RSCRIPT=/usr/bin/Rscript
 
 # set make arguments to build using multiple jobs
 export MAKE_ARGS=-j
+
+export TEST_BASE=/met
+export COMPILER=gnu
+export MET_SUBDIR=${TEST_BASE}
+export MET_TARBALL=none
+export USE_MODULES=FALSE
+
+export MET_INSTALL_DIR=/usr/local
+
+export COMPILE_HDF=1
+export COMPILE_HDFEOS=1
+export COMPILE_FREETYPE=1
+export COMPILE_CAIRO=1
diff --git a/internal/scripts/installation/compile_MET_all.sh b/internal/scripts/installation/compile_MET_all.sh
@@ -89,8 +89,23 @@ echo "MET_TARBALL = ${MET_TARBALL? "ERROR: MET_TARBALL must be set"}"
 echo "USE_MODULES = ${USE_MODULES? "ERROR: USE_MODULES must be set to TRUE if using modules or FALSE otherwise"}"
 echo ${MAKE_ARGS:+MAKE_ARGS = $MAKE_ARGS}
 
-export LIB_DIR=${TEST_BASE}/external_libs
+LIB_DIR=${TEST_BASE}/external_libs
 MET_DIR=${MET_SUBDIR}
+
+if [ -z "${BIN_DIR_PATH}" ]; then
+  if [ -z "${MET_INSTALL_DIR}" ]; then
+    BIN_DIR_PATH=${TEST_BASE}/bin
+  else
+    BIN_DIR_PATH=${MET_INSTALL_DIR}/bin
+  fi
+fi
+
+if [ -z "${MET_INSTALL_DIR}" ]; then
+  MET_INSTALL_DIR=${MET_DIR}
+else
+  LIB_DIR=${MET_INSTALL_DIR}
+fi
+
 TAR_DIR=${TEST_BASE}/tar_files
 MET_TARBALL=${TAR_DIR}/${MET_TARBALL}
 
@@ -415,12 +430,16 @@ if [ $COMPILE_G2CLIB -eq 1 ]; then
   rm -rf ${LIB_DIR}/g2clib/g2clib*
   tar -xf ${TAR_DIR}/g2clib*.tar -C ${LIB_DIR}/g2clib
   cd ${LIB_DIR}/g2clib/g2clib*
-  cat makefile | \
-    sed -r 's/INC=.*/INC=-I${LIB_DIR}\/include -I${LIB_DIR}\/include\/jasper/g' | \
-    sed 's/CC=gcc/CC=${CC_COMPILER}/g' | \
-    sed 's/-D__64BIT__//g' \
-    > makefile_new
-  mv makefile_new makefile
+  sed -i 's|INC=.*|INC=-I${LIB_DIR}/include -I${LIB_DIR}/include/jasper|g' makefile
+
+  # allow other compilers besides gcc
+  sed -i 's/CC=gcc/CC=${CC_COMPILER}/g' makefile
+
+  # remove -D__64BIT__ flag because compiling with it has
+  # shown issues with GRIB/GRIB2 files that are over 2GB in size
+  # This flag was removed in g2clib 1.6.4
+  # so this can be removed if the version is updated
+  sed -i 's/-D__64BIT__//g' makefile
   export CC_COMPILER=${CC}
   echo "cd `pwd`"
   # g2clib appears to compile but causes failure compiling MET if -j argument is used
@@ -628,7 +647,7 @@ if [[ $COMPILER_FAMILY == "pgi" ]]; then
   export OPT_ARGS="${OPT_ARGS} FFLAGS=-lpgf90"
 fi
 
-configure_cmd="./configure --prefix=${MET_DIR} --bindir=${BIN_DIR_PATH}"
+configure_cmd="./configure --prefix=${MET_INSTALL_DIR} --bindir=${BIN_DIR_PATH}"
 configure_cmd="${configure_cmd} BUFRLIB_NAME=${BUFRLIB_NAME}"
 configure_cmd="${configure_cmd} GRIB2CLIB_NAME=${GRIB2CLIB_NAME} --enable-grib2"
 if [[ ! -z ${MET_FREETYPEINC} && ! -z ${MET_FREETYPELIB} && \

diff --git a/internal/test_unit/config/PointStatConfig_climo b/internal/test_unit/config/PointStatConfig_climo
@@ -53,21 +53,24 @@ fcst = {
 
    field = [
       {
-        name         = "TMP";
-        level        = [ "Z2" ];
-        desc         = "KELVIN";
+        name           = "TMP";
+        level          = [ "Z2" ];
+        desc           = "KELVIN";
+        set_attr_units = "K";
       },
       {
-        name         = "TMP";
-        level        = [ "Z2" ];
-        desc         = "CELCIUS";
-        convert(x)   = K_to_C(x);
+        name           = "TMP";
+        level          = [ "Z2" ];
+        desc           = "CELCIUS";
+        convert(x)     = K_to_C(x);
+        set_attr_units = "C";
       },
       {
-        name         = "TMP";
-        level        = [ "Z2" ];
-        desc         = "FAHRENHEIT";
-        convert(x)   = K_to_F(x);
+        name           = "TMP";
+        level          = [ "Z2" ];
+        desc           = "FAHRENHEIT";
+        convert(x)     = K_to_F(x);
+        set_attr_units = "F";
       },
       {
         name         = "TMP";

diff --git a/internal/test_unit/xml/unit_climatology_2.5deg.xml b/internal/test_unit/xml/unit_climatology_2.5deg.xml
@@ -81,4 +81,33 @@
     </output>
   </test>
 
+  <!-- MET#2437: Test interpolation with a single input monthly climo file -->
+
+  <test name="climatology_GRID_STAT_SINGLE_MONTH_2.5DEG">
+    <exec>&MET_BIN;/grid_stat</exec>
+    <env>
+      <pair><name>OUTPUT_PREFIX</name> <value>SINGLE_MONTH_CLIMO_2.5DEG</value></pair>
+      <pair><name>DAY_INTERVAL</name>  <value>31</value></pair>
+      <pair><name>HOUR_INTERVAL</name> <value>6</value></pair>
+      <pair><name>CLIMO_MEAN_FILE_LIST</name>
+         <value>"&DATA_DIR_CLIMO;/NCEP_2.5deg/pgba_mean.19591215"
+         </value>
+      </pair>
+      <pair><name>CLIMO_STDEV_FILE_LIST</name>
+         <value>"&DATA_DIR_CLIMO;/NCEP_2.5deg/pgba_stdv.19591215"
+         </value>
+      </pair>
+    </env>
+    <param> \
+      &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120409_1200_000.grb2 \
+      &DATA_DIR_MODEL;/grib2/gfsanl/gfsanl_4_20120409_1200_000.grb2 \
+      &CONFIG_DIR;/GridStatConfig_climo_wrap_year \
+      -outdir &OUTPUT_DIR;/climatology_2.5deg -v 3
+    </param>
+    <output>
+      <stat>&OUTPUT_DIR;/climatology_2.5deg/grid_stat_SINGLE_MONTH_CLIMO_2.5DEG_000000L_20201225_120000V.stat</stat>
+      <grid_nc>&OUTPUT_DIR;/climatology_2.5deg/grid_stat_SINGLE_MONTH_CLIMO_2.5DEG_000000L_20201225_120000V_pairs.nc</grid_nc>
+    </output>
+  </test>
+
 </met_test>
diff --git a/scripts/config/GridStatConfig_all b/scripts/config/GridStatConfig_all
@@ -107,7 +107,6 @@ climo_mean = {
    }
 
    time_interp_method = DW_MEAN;
-   match_month        = TRUE;
    match_day          = FALSE;
    time_step          = 21600;
 }

diff --git a/src/basic/vx_cal/doyhms_to_unix.cc b/src/basic/vx_cal/doyhms_to_unix.cc
@@ -178,3 +178,4 @@ return ( dt );
 
 
 ////////////////////////////////////////////////////////////////////////
+
diff --git a/src/basic/vx_config/config_util.cc b/src/basic/vx_config/config_util.cc
@@ -1299,14 +1299,18 @@ RegridInfo parse_conf_regrid(Dictionary *dict, bool error_out) {
    info.gaussian.trunc_factor = (is_bad_data(conf_value) ? default_trunc_factor : conf_value);
    if (info.method == InterpMthd_Gaussian || info.method == InterpMthd_MaxGauss) info.gaussian.compute();
 
+   // MET#2437 Do not search the higher levels of config file context for convert,
+   //          censor_thresh, and censor_val. They must be specified within the
+   //          regrid dictionary itself.
+
    // Conf: convert
-   info.convert_fx.set(regrid_dict->lookup(conf_key_convert));
+   info.convert_fx.set(regrid_dict->lookup(conf_key_convert, false));
 
    // Conf: censor_thresh
-   info.censor_thresh = regrid_dict->lookup_thresh_array(conf_key_censor_thresh, false);
+   info.censor_thresh = regrid_dict->lookup_thresh_array(conf_key_censor_thresh, false, true, false);
 
    // Conf: censor_val
-   info.censor_val = regrid_dict->lookup_num_array(conf_key_censor_val, false);
+   info.censor_val = regrid_dict->lookup_num_array(conf_key_censor_val, false, true, false);
 
    // Validate the settings
    info.validate();