Update develop-ref after dtcenter/MET#2655 and #2311 (#2331)

Co-authored-by: Lisa Goodrich <lisag@ucar.edu> Co-authored-by: Julie Prestopnik <jpresto@seneca.rap.ucar.edu> Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com> Co-authored-by: Hank Fisher <fisherh@ucar.edu> Co-authored-by: Dan Adriaansen <dadriaan@ucar.edu> Co-authored-by: johnhg <johnhg@ucar.edu> Co-authored-by: John Halley Gotway <johnhg@ucar.edu> Co-authored-by: jprestop <jpresto@ucar.edu> Co-authored-by: Tracy Hertneky <hertneky@seneca.rap.ucar.edu> Co-authored-by: Giovanni Rosa <giovanni.rosa@unimol.it> Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com> Co-authored-by: Mrinal Biswas <biswas@seneca.rap.ucar.edu> Co-authored-by: j-opatz <jopatz@ucar.edu> Co-authored-by: Daniel Adriaansen <dadriaan@ucar.edu> Co-authored-by: Jonathan Vigh <jvigh@ucar.edu> Co-authored-by: root <root@localhost> Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com> Co-authored-by: Will Mayfield <59745143+willmayfield@users.noreply.github.com> Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com> Co-authored-by: metplus-bot <97135045+metplus-bot@users.noreply.github.com> Co-authored-by: Tracy Hertneky <39317287+hertneky@users.noreply.github.com> Co-authored-by: Giovanni Rosa <g.rosa1@studenti.unimol.it> Co-authored-by: mrinalbiswas <biswas@ucar.edu> Co-authored-by: Christina Kalb <kalb@ucar.edu> Co-authored-by: jason-english <73247785+jason-english@users.noreply.github.com> Co-authored-by: John Sharples <41682323+John-Sharples@users.noreply.github.com> fix GitHub Actions warnings (#1864) fix #1884 develop PCPCombine {custom} in subtract method (#1887) fix #1939 develop - failure reading obs when zipped file also exists (#1941) Closes #1986 fix develop Fix broken documentation links (#2004) fix #2026 develop StatAnalysis looping (#2028) fix priority of obs_window config variables so that wrapper-specific version is preferred over generic OBS_WINDOW_BEGIN/END (#2062) fix #2070 var list numeric order (#2072) fix #2087 develop docs_pdf (#2091) fix #2096/#2098 develop - fix skip if output exists and do not error if no commands were run (#2099) Fix for Dockerfile smell DL4000 (#2112) fix #2082 develop regrid.convert/censor_thresh/censor_val (#2140) fix #2082 main_v5.0 regrid.convert/censor_thresh/censor_val (#2101) fix #2137 develop PointStat -obs_valid_beg/end (#2141) fix failured introduced by urllib3 (see urllib3/urllib3#2168) fix #2161 develop PCPCombine additional field arguments in -subtract mode (#2162) fix #2168 develop - StatAnalysis time shift (#2169) fix releases. (#2183) fix #2189 develop - spaces in complex thresholds (#2191) fix #2179 develop TCPairs fix -diag argument (#2187) fixes (#2200) fix diff tests (#2217) fix automated tests (#2237) fix #2235 rename multivar_itensity to multivar_intensity_flag (#2236) fix #2241 Create directory containing -out_stat file (#2242) fix #2245 use unique run ID to name logger instance (#2247) fix #2244 develop fix diff tests (#2254) fixture to set pytest tmpdir (#2261) fix #1853 develop - PointStat don't require mask variables to be set (#2262) fix #2279 develop - buoy station file from 2022 (#2280)
dtcenter · Aug 30, 2023 · 9a3864f · 9a3864f
1 parent af49567
commit 9a3864f
Show file tree

Hide file tree

Showing 101 changed files with 1,901 additions and 1,819 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,3 @@
+[run]
+relative_files = True
+source = metplus
diff --git a/.github/actions/run_tests/entrypoint.sh b/.github/actions/run_tests/entrypoint.sh
@@ -12,7 +12,7 @@ source ${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/bash_functions.sh
 
 # get branch name for push or pull request events
 # add -pull_request if pull request event to keep separated
-branch_name=`${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/print_branch_name.py`
+branch_name=$(${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/print_branch_name.py)
 if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
   branch_name=${branch_name}-pull_request
 fi
@@ -23,57 +23,25 @@ time_command docker pull $DOCKERHUBTAG
 
 # if unsuccessful (i.e. pull request from a fork)
 # then build image locally
-docker inspect --type=image $DOCKERHUBTAG > /dev/null
-if [ $? != 0 ]; then
+if ! docker inspect --type=image $DOCKERHUBTAG > /dev/null; then
    # if docker pull fails, build locally
    echo docker pull failed. Building Docker image locally...
    ${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/docker_setup.sh
 fi
 
-# running unit tests (pytests)
-if [[ "$INPUT_CATEGORIES" == pytests* ]]; then
-  export METPLUS_ENV_TAG="test.v5.1"
-  export METPLUS_IMG_TAG=${branch_name}
-  echo METPLUS_ENV_TAG=${METPLUS_ENV_TAG}
-  echo METPLUS_IMG_TAG=${METPLUS_IMG_TAG}
-
-  export RUN_TAG=metplus-run-env
-
-  # use BuildKit to build image
-  export DOCKER_BUILDKIT=1
-
-  start_seconds=$SECONDS
-
-  # build an image with the pytest conda env and the METplus branch image
-  # Note: adding --build-arg <arg-name> without any value tells docker to
-  #  use value from local environment (export METPLUS_IMG_TAG)
-  time_command docker build -t $RUN_TAG \
-	 --build-arg METPLUS_IMG_TAG \
-	 --build-arg METPLUS_ENV_TAG \
-	 -f .github/actions/run_tests/Dockerfile.run \
-	 .
-
-  echo Running Pytests
-  command="export METPLUS_TEST_OUTPUT_BASE=/data/output;"
-  command+="/usr/local/conda/envs/${METPLUS_ENV_TAG}/bin/pytest internal/tests/pytests -vv --cov=metplus --cov-append --cov-report=term-missing;"
-  command+="if [ \$? != 0 ]; then echo ERROR: Some pytests failed. Search for FAILED to review; false; fi"
-  time_command docker run -v $WS_PATH:$GITHUB_WORKSPACE --workdir $GITHUB_WORKSPACE $RUN_TAG bash -c "$command"
-  exit $?
-fi
-
 # running use case tests
 
 # split apart use case category and subset list from input
-CATEGORIES=`echo $INPUT_CATEGORIES | awk -F: '{print $1}'`
-SUBSETLIST=`echo $INPUT_CATEGORIES | awk -F: '{print $2}'`
+CATEGORIES=$(echo $INPUT_CATEGORIES | awk -F: '{print $1}')
+SUBSETLIST=$(echo $INPUT_CATEGORIES | awk -F: '{print $2}')
 
 # run all cases if no subset list specified
 if [ -z "${SUBSETLIST}" ]; then
     SUBSETLIST="all"
 fi
 
 # get METviewer if used in any use cases
-all_requirements=`./${CI_JOBS_DIR}/get_requirements.py ${CATEGORIES} ${SUBSETLIST}`
+all_requirements=$(./${CI_JOBS_DIR}/get_requirements.py ${CATEGORIES} ${SUBSETLIST})
 echo All requirements: $all_requirements
 NETWORK_ARG=""
 if [[ "$all_requirements" =~ .*"metviewer".* ]]; then

diff --git a/.github/jobs/get_use_cases_to_run.sh b/.github/jobs/get_use_cases_to_run.sh
@@ -7,11 +7,9 @@ matrix="[]"
 
 run_use_cases=$1
 run_all_use_cases=$2
-run_unit_tests=$3
 
 echo Run use cases: $run_use_cases
 echo Run all use cases: $run_all_use_cases
-echo Run unit tests: $run_unit_tests
 
 # if running use cases, generate JQ filter to use
 if [ "$run_use_cases" == "true" ]; then
@@ -28,21 +26,6 @@ if [ "$run_use_cases" == "true" ]; then
 
 fi
 
-# if unit tests will be run, add "pytests" to beginning of matrix list
-if [ "$run_unit_tests" == "true" ]; then
-  echo Adding unit tests to list to run
-
-  pytests="\"pytests\","
-
-  # if matrix is empty, set to an array that only includes pytests
-  if [ "$matrix" == "[]" ]; then
-    matrix="[${pytests:0: -1}]"
-  # otherwise prepend item to list
-  else
-    matrix="[${pytests}${matrix:1}"
-  fi
-fi
-
 echo Array of groups to run is: $matrix
 # if matrix is still empty, exit 1 to fail step and skip rest of workflow
 if [ "$matrix" == "[]" ]; then

diff --git a/.github/jobs/set_job_controls.sh b/.github/jobs/set_job_controls.sh
@@ -87,6 +87,7 @@ fi
 echo "run_get_image=$run_get_image" >> $GITHUB_OUTPUT
 echo "run_get_input_data=$run_get_input_data" >> $GITHUB_OUTPUT
 echo "run_diff=$run_diff" >> $GITHUB_OUTPUT
+echo "run_unit_tests=$run_unit_tests" >> $GITHUB_OUTPUT
 echo "run_save_truth_data=$run_save_truth_data" >> $GITHUB_OUTPUT
 echo "external_trigger=$external_trigger" >> $GITHUB_OUTPUT
 
@@ -96,7 +97,7 @@ branch_name=`${GITHUB_WORKSPACE}/.github/jobs/print_branch_name.py`
 echo "branch_name=$branch_name" >> $GITHUB_OUTPUT
 
 # get use cases to run
-.github/jobs/get_use_cases_to_run.sh $run_use_cases $run_all_use_cases $run_unit_tests
+.github/jobs/get_use_cases_to_run.sh $run_use_cases $run_all_use_cases
 
 # echo output variables to review in logs
 echo branch_name: $branch_name

diff --git a/.github/labels/common_labels.txt b/.github/labels/common_labels.txt
@@ -7,7 +7,9 @@
 {"name": "component: documentation","color": "1d76db","description": "Documentation issue"}
 {"name": "component: external dependency","color": "1d76db","description": "External dependency issue"}
 {"name": "component: code optimization","color": "1d76db","description": "Code optimization issue"}
+{"name": "component: input data","color": "1d76db","description": "Input data issue"}
 {"name": "component: release engineering","color": "1d76db","description": "Release engineering issue"}
+{"name": "component: repository maintenance","color": "1d76db","description": "Repository maintenance issue"}
 {"name": "component: testing","color": "1d76db","description": "Software testing issue"}
 {"name": "component: training","color": "1d76db","description": "Training issue"}
 {"name": "component: user support","color": "1d76db","description": "User support issue"}

diff --git a/.github/labels/delete_labels.sh b/.github/labels/delete_labels.sh
@@ -15,7 +15,10 @@ else
   repo=$3
 fi
 
-# Constants
+# Verbose output
+VERBOSE=0
+
+# GitHub label URL
 URL="https://api.github.com/repos/dtcenter/${repo}/labels"
 COMMON_LABELS="`dirname $0`/common_labels.txt"
 
@@ -28,27 +31,43 @@ SCRIPT_DIR=`dirname $0`
 TMP_FILE="${repo}_labels.tmp"
 CMD="${SCRIPT_DIR}/get_labels.sh ${user} ${auth} ${repo}"
 echo "CALLING: ${CMD}"
-${CMD} > ${TMP_FILE}
+${CMD} > ${TMP_FILE} 2>/dev/null
+
+# Initialize counts
+n_common=0
+n_custom=0
+n_delete=0
 
 # Check each of the existing labels against the common list
 while read -r line; do
 
   # Parse the label name
   name=`echo $line | sed -r 's/,/\n/g' | grep '"name":' | cut -d':' -f2-10 | cut -d'"' -f2`
 
-  # Check if it's a common label and a component label
+  # Check if it appears in the list of common labels
   is_common=`egrep -i "\"${name}\"" ${COMMON_LABELS} | wc -l`
-  is_custom=`echo ${name} | egrep "component:|type:" | wc -l`
+
+  # Check if its a custom label that beginning with component, type, or repository name
+  is_custom=`echo ${name} | egrep -r -i "component:|type:|${repo}" | wc -l`
 
   # Keep COMMON labels
   if [[ $is_common -gt 0 ]]; then
-    echo "[COMMON] ${repo} label ... ${name}"
+    ((n_common+=1))
+    if [[ $VERBOSE -gt 0 ]]; then
+      echo "[COMMON] ${repo} label ... ${name}"
+    fi
   # Keep CUSTOM, repo-specific labels
   elif [[ $is_custom -gt 0 ]]; then
-    echo "[CUSTOM] ${repo} label ... ${name}"
+    ((n_custom+=1))
+    if [[ $VERBOSE -gt 0 ]]; then
+      echo "[CUSTOM] ${repo} label ... ${name}"
+    fi
   # DELETE non-common, non-custom labels
-  else 
-    echo "[DELETE] ${repo} label ... ${name}"
+  else
+    ((n_delete+=1))
+    if [[ $VERBOSE -gt 0 ]]; then
+      echo "[DELETE] ${repo} label ... ${name}"
+    fi
     DELETE_URL="${URL}/`echo ${name} | sed -r 's/ /%20/g'`"
     echo "curl -u \"${user}:${auth}\" -X DELETE \
           -H \"Accept: application/vnd.github.v3+json\" \
@@ -62,5 +81,8 @@ rm -f ${TMP_FILE}
 
 # Make the run command file executable
 chmod +x ${CMD_FILE}
-echo "To make these changes, execute the run command file:"
-echo "./${CMD_FILE}"
+
+# Print summary
+echo "For the ${repo} repository, found $n_common common, $n_custom custom, and $n_delete labels to be deleted."
+echo "To delete $n_delete existing ${repo} labels, run:"
+echo "  ${CMD_FILE}"
diff --git a/.github/labels/get_labels.sh b/.github/labels/get_labels.sh
@@ -16,8 +16,11 @@ else
 fi
 
 # Pull and format existing records for existing labels
-curl -u "${user}:${auth}" -H "Accept: application/vnd.github.v3+json" \
-"https://api.github.com/repos/dtcenter/${repo}/labels?page=1&per_page=100" | \
-egrep  '"name":|"color":|"description":|{|}' | \
-tr -d '\n' | sed -r 's/ +/ /g' | sed 's/}/}\n/g' | sed 's/,* {/{/g'
+# Run twice for page 1 and page 2 to support up to 200 existing labels
+for page_number in 1 2; do
+  curl -u "${user}:${auth}" -H "Accept: application/vnd.github.v3+json" \
+  "https://api.github.com/repos/dtcenter/${repo}/labels?page=${page_number}&per_page=100" | \
+  egrep  '"name":|"color":|"description":|{|}' | \
+  tr -d '\n' | sed -r 's/ +/ /g' | sed 's/}/}\n/g' | sed 's/,* {/{/g'
+done
 
diff --git a/.github/labels/post_patch_labels.sh b/.github/labels/post_patch_labels.sh
@@ -17,19 +17,29 @@ else
   labels=$4
 fi
 
+# Verbose output
+VERBOSE=0
+
 # GitHub label URL
 URL="https://api.github.com/repos/dtcenter/${repo}/labels"
 
-# Output command file
-CMD_FILE="`dirname $0`/commands/post_patch_labels_${repo}_cmd.sh"
-echo "#!/bin/sh -v" > ${CMD_FILE}
+# Output command files
+POST_CMD_FILE="`dirname $0`/commands/post_labels_${repo}_cmd.sh"
+echo "#!/bin/sh -v" > ${POST_CMD_FILE}
+
+PATCH_CMD_FILE="`dirname $0`/commands/patch_labels_${repo}_cmd.sh"
+echo "#!/bin/sh -v" > ${PATCH_CMD_FILE}
+
+# Initialize counts
+n_post=0
+n_patch=0
 
 # Get the current repo labels
 SCRIPT_DIR=`dirname $0`
 TMP_FILE="${repo}_labels.tmp"
 CMD="${SCRIPT_DIR}/get_labels.sh ${user} ${auth} ${repo}"
 echo "CALLING: ${CMD}"
-${CMD} > ${TMP_FILE}
+${CMD} > ${TMP_FILE} 2>/dev/null
 
 # Read the lines of the label file
 while read -r line; do
@@ -42,17 +52,23 @@ while read -r line; do
 
   # POST a new label
   if [[ $exists -eq 0 ]]; then
-    echo "[POST ] ${repo} label ... ${name}"
+    ((n_post+=1))
+    if [[ $VERBOSE -gt 0 ]]; then
+      echo "[POST ] ${repo} label ... ${name}"
+    fi
     echo "curl -u \"${user}:${auth}\" -X POST \
           -H \"Accept: application/vnd.github.v3+json\" \
-          -d '${line}' '${URL}'" >> ${CMD_FILE}
+          -d '${line}' '${URL}'" >> ${POST_CMD_FILE}
   # PATCH an existing label
   else
+    ((n_patch+=1))
     old_name=`egrep -i "\"${name}\"" ${TMP_FILE} | sed -r 's/,/\n/g' | grep '"name":' | cut -d':' -f2-10 | cut -d'"' -f2 | sed -r 's/ /%20/g'`
-    echo "[PATCH] ${repo} label ... ${old_name} -> ${name}"
+    if [[ $VERBOSE -gt 0 ]]; then
+      echo "[PATCH] ${repo} label ... ${old_name} -> ${name}"
+    fi
     echo "curl -u \"${user}:${auth}\" -X PATCH \
           -H \"Accept: application/vnd.github.v3+json\" \
-          -d '${line}' '${URL}/${old_name}'" >> ${CMD_FILE}
+          -d '${line}' '${URL}/${old_name}'" >> ${PATCH_CMD_FILE}
   fi
 
 done < $labels
@@ -61,7 +77,11 @@ done < $labels
 rm -f ${TMP_FILE}
 
 # Make the run command file executable
-chmod +x ${CMD_FILE}
-echo "To make these changes, execute the run command file:"
-echo "./${CMD_FILE}"
+chmod +x ${POST_CMD_FILE} ${PATCH_CMD_FILE}
 
+# Print summary
+echo "For the ${repo} repository, found $n_patch existing labels to be updated and $n_post new labels to be added."
+echo "To add $n_post new ${repo} labels, run:"
+echo "  ${POST_CMD_FILE}"
+echo "To update $n_patch existing ${repo} labels, run:"
+echo "  ${PATCH_CMD_FILE}"
diff --git a/.github/labels/process_labels.sh b/.github/labels/process_labels.sh
@@ -19,14 +19,21 @@ SCRIPT_DIR=`dirname $0`
 REPO_LIST="metplus met metplotpy metcalcpy metdataio metviewer \
            metexpress metplus-training";
 
-# Build commands to add/update common labels
+# Process each repository
 for REPO in ${REPO_LIST}; do
-  echo $REPO
-  ${SCRIPT_DIR}/post_patch_labels.sh $USER $AUTH $REPO ${SCRIPT_DIR}/common_labels.txt
-done
 
-# Build commands to delete extra labels
-for REPO in ${REPO_LIST}; do
-  echo $REPO;
-  ${SCRIPT_DIR}/delete_labels.sh $USER $AUTH $REPO
+  echo
+  echo "Processing repository: ${REPO}"
+  echo
+
+  # Build commands to add/update common labels
+  CMD="${SCRIPT_DIR}/post_patch_labels.sh $USER $AUTH $REPO ${SCRIPT_DIR}/common_labels.txt"
+  echo "CALLING: ${CMD}"
+  ${CMD}
+
+  # Build commands to delete extra labels
+  CMD="${SCRIPT_DIR}/delete_labels.sh $USER $AUTH $REPO"
+  echo "CALLING: ${CMD}"
+  ${CMD}
+
 done
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -23,6 +23,6 @@ See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors
 Select: **Reviewer(s)**
 Select: **Organization** level software support **Project** or **Repository** level development cycle **Project**
 Select: **Milestone** as the version that will include these changes
-- [ ] fter submitting the PR, select the :gear: icon in the **Development** section of the right hand sidebar. Search for the issue that this PR will close and select it, if it is not already selected.
+- [ ] After submitting the PR, select the :gear: icon in the **Development** section of the right hand sidebar. Search for the issue that this PR will close and select it, if it is not already selected.
 - [ ] After the PR is approved, merge your changes. If permissions do not allow this, request that the reviewer do the merge.
 - [ ] Close the linked issue and delete your feature or bugfix branch from GitHub.