Feature 2717 convert unit.pl to unit.py (#2871)

* created unit.py module in new internal/test_unit/python directory

* added xml parsing to unit.py

* added repl_env function

* added reading of the remaining xml tags in build_tests function

* progress on main function (putting together test commands)

* a few more lines in the main function

* minor updates

* fixed how the test command was being run

* added if name/main and command line parsing

* fixed handling of no 'env' in cmd_only mode

* handle params from xml that have \ after filename without space in between

* added logging

* added some more pieces to unit

* more updates to unit.py, including running checks on output files

* bug fixes, improved handling of output file names, improved handling of env vars, improved logging output

* fixed how shell commands are run, and other minor fixes

* added last bits from the perl script, fixed some bugs

* created unit.py module in new internal/test_unit/python directory

* added xml parsing to unit.py

* added repl_env function

* added reading of the remaining xml tags in build_tests function

* progress on main function (putting together test commands)

* a few more lines in the main function

* minor updates

* update scripts to call python unit test script instead of the old perl script

* fix she-bang line to allow script to be run without python3 before it

* add missing test_dir and exit_on_fail tags that are found in the rest of the unit test xml files

* fix call to logger.warning

* change tags named 'exists' to 'exist' to match the rest of the xml files

* added logger to function

* removed tab at end of line that was causing output file path to be excluded from the command

* fix broken checks for output files

* incorporated george's recommended changes

* changed default to overwrite logs; allow for more than one xml file to be passed in command

---------

Co-authored-by: Natalie babij <natalie.babij@natbabij-mac1.home>
Co-authored-by: Natalie babij <natalie.babij@vpn68-237.vpnclient.ucar.edu>
Co-authored-by: Natalie babij <natalie.babij@gsd-sslvpn-209.fsl.noaa.gov>
Co-authored-by: Natalie Babij <nbabij@seneca.rap.ucar.edu>
Co-authored-by: John Halley Gotway <johnhg@ucar.edu>
Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Co-authored-by: j-opatz <jopatz@ucar.edu>

.github/jobs/run_unit_tests.sh

@@ -15,7 +15,7 @@ source ${MET_REPO_DIR}/.github/jobs/test_env_vars.sh
 echo "Running MET unit tests..."
 for testname in $TESTS_TO_RUN; do
   CMD_LOGFILE=/met/logs/unit_${testname}.log
-  time_command ${MET_TEST_BASE}/perl/unit.pl ${MET_TEST_BASE}/xml/unit_${testname}.xml
+  time_command ${MET_TEST_BASE}/python/unit.py ${MET_TEST_BASE}/xml/unit_${testname}.xml
   if [ $? != 0 ]; then
       echo "ERROR: Unit test ${testname} failed"
       cat /met/logs/unit_${testname}.log

internal/test_unit/bin/unit_test.sh

@@ -24,14 +24,14 @@ if [[ -z "${MET_TEST_MET_PYTHON_EXE}" ]] ; then
   export MET_TEST_MET_PYTHON_EXE=/usr/local/python3/bin/python3
 fi
 
-PERL_UNIT_OPTS=""
+UNIT_OPTS=""
 for arg in $@; do
-  [ $arg == "-memchk" -o $arg == "memchk" ] && PERL_UNIT_OPTS="$PERL_UNIT_OPTS -memchk"
-  [ $arg == "-callchk" -o $arg == "callchk" ] && PERL_UNIT_OPTS="$PERL_UNIT_OPTS -callchk"
+  [ $arg == "-memchk" -o $arg == "memchk" ] && UNIT_OPTS="$UNIT_OPTS -memchk"
+  [ $arg == "-callchk" -o $arg == "callchk" ] && UNIT_OPTS="$UNIT_OPTS -callchk"
 done
 
 # Unit test script
-PERL_UNIT=${MET_TEST_BASE}/perl/unit.pl
+UNIT=${MET_TEST_BASE}/python/unit.py
 
 # Unit test XML
 UNIT_XML="unit_ascii2nc.xml \
@@ -107,15 +107,15 @@ UNIT_XML="${UNIT_XML} unit_ugrid.xml"
 for CUR_XML in ${UNIT_XML}; do
 
   echo
-  echo "CALLING: ${PERL_UNIT} $PERL_UNIT_OPTS ${MET_TEST_BASE}/xml/${CUR_XML}"
+  echo "CALLING: ${UNIT} $UNIT_OPTS ${MET_TEST_BASE}/xml/${CUR_XML}"
   echo
-  ${PERL_UNIT} $PERL_UNIT_OPTS ${MET_TEST_BASE}/xml/${CUR_XML}
+  ${UNIT} $UNIT_OPTS ${MET_TEST_BASE}/xml/${CUR_XML}
   RET_VAL=$?
 
   # Fail on non-zero return status
   if [ ${RET_VAL} != 0 ]; then
     echo
-    echo "ERROR: ${PERL_UNIT} ${CUR_XML} failed."
+    echo "ERROR: ${UNIT} ${CUR_XML} failed."
     echo
     echo "*** UNIT TESTS FAILED ***"
     echo