diff --git a/.github/actions/run_tests/entrypoint.sh b/.github/actions/run_tests/entrypoint.sh index 7e0a4b72d7..7dda2d2133 100644 --- a/.github/actions/run_tests/entrypoint.sh +++ b/.github/actions/run_tests/entrypoint.sh @@ -56,7 +56,7 @@ if [[ "$INPUT_CATEGORIES" == pytests* ]]; then . echo Running Pytests - command="export METPLUS_PYTEST_HOST=docker; cd internal_tests/pytests;" + command="export METPLUS_PYTEST_HOST=docker; cd internal/tests/pytests;" command+="status=0;" for x in `cat $PYTESTS_GROUPS_FILEPATH`; do marker="${x//_or_/ or }" diff --git a/.github/jobs/get_requirements.py b/.github/jobs/get_requirements.py index e4419e31d1..661df3f5ac 100755 --- a/.github/jobs/get_requirements.py +++ b/.github/jobs/get_requirements.py @@ -14,7 +14,7 @@ os.pardir)) sys.path.insert(0, METPLUS_TOP_DIR) -from internal_tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite +from internal.tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite def main(): all_requirements = set() diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 3d8956181b..04920faa9c 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -13,7 +13,7 @@ os.pardir)) sys.path.insert(0, METPLUS_TOP_DIR) -from internal_tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite +from internal.tests.use_cases.metplus_use_case_suite import METplusUseCaseSuite from metplus.util.met_util import expand_int_string_to_list from docker_utils import VERSION_EXT diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst index cad0bdef45..cf47101882 100644 --- a/docs/Contributors_Guide/add_use_case.rst +++ b/docs/Contributors_Guide/add_use_case.rst @@ -680,7 +680,7 @@ will be used in the final pull request. Add use case to the test suite ------------------------------ -The *internal_tests/use_cases/all_use_cases.txt* file in the METplus +The *internal/tests/use_cases/all_use_cases.txt* file in the METplus repository contains the list of all use cases. Add the new use case to this file so it will be available in the tests. See the :ref:`cg-ci-all-use-cases` section for details. @@ -848,7 +848,7 @@ steps were unsuccessful in lowering memory usage, please take the following step Utilize a Python memory profiler to identify as specifically as possible where the script exceeds the memory limit. - Add the use case to the :ref:`memory-intense-use-cases` list. -- In the *internal_tests/use_cases/all_use_cases.txt* file, ensure that the +- In the *internal/tests/use_cases/all_use_cases.txt* file, ensure that the use case is listed as the lowest-listed use case in its respective category. Change the number in front of the new use case to an 'X', preceded by the ‘#’ character:: diff --git a/docs/Contributors_Guide/continuous_integration.rst b/docs/Contributors_Guide/continuous_integration.rst index e3fb5ccfe1..bed495e39c 100644 --- a/docs/Contributors_Guide/continuous_integration.rst +++ b/docs/Contributors_Guide/continuous_integration.rst @@ -601,7 +601,7 @@ All Use Cases ^^^^^^^^^^^^^ All of the existing use cases are listed in **all_use_cases.txt**, -found in *internal_tests/use_cases*. +found in *internal/tests/use_cases*. The file is organized by use case category. Each category starts a line that following the format:: diff --git a/docs/Contributors_Guide/testing.rst b/docs/Contributors_Guide/testing.rst index 93c2f9627f..e1e0af8e6a 100644 --- a/docs/Contributors_Guide/testing.rst +++ b/docs/Contributors_Guide/testing.rst @@ -1,7 +1,7 @@ Testing ======= -Test scripts are found in the GitHub repository in the internal_tests +Test scripts are found in the GitHub repository in the internal/tests directory. .. _cg-unit-tests: @@ -13,7 +13,7 @@ Unit tests are run with pytest. They are found in the *pytests* directory. Each tool has its own subdirectory containing its test files. Unit tests can be run by running the 'pytest' command from the -internal_tests/pytests directory of the repository. +internal/tests/pytests directory of the repository. The 'pytest' Python package must be available. A report will be output showing which pytest categories failed. When running on a new computer, a **minimum_pytest..sh** @@ -21,7 +21,7 @@ file must be created to be able to run the script. This file contains information about the local environment so that the tests can run. All unit tests must include one of the custom markers listed in the -internal_tests/pytests/pytest.ini file. Some examples include: +internal/tests/pytests/pytest.ini file. Some examples include: * util * wrapper_a diff --git a/docs/Release_Guide/release_steps/metplus/update_web_server_data.rst b/docs/Release_Guide/release_steps/metplus/update_web_server_data.rst index f25205ded3..86ccbde0c3 100644 --- a/docs/Release_Guide/release_steps/metplus/update_web_server_data.rst +++ b/docs/Release_Guide/release_steps/metplus/update_web_server_data.rst @@ -7,7 +7,7 @@ Create Directory for Next Release On the DTC web server where the sample input data for use cases is hosted, run the setup_next_release_data.py script for the next upcoming release to set up the data directory for the next major/minor version development. -The script can be found in the METplus repository in internal_tests/use_cases. +The script can be found in the METplus repository in internal/tests/use_cases. The file should be found in the home directory of the met_test user on the DTC web server host. It is linked to the file in the METplus repository. Pull the latest changes from the develop branch before running the script:: diff --git a/docs/Users_Guide/installation.rst b/docs/Users_Guide/installation.rst index 9f5bd3fbbe..a935998ab7 100644 --- a/docs/Users_Guide/installation.rst +++ b/docs/Users_Guide/installation.rst @@ -227,7 +227,7 @@ The METplus Wrappers source code contains the following directory structure:: build_components/ docs/ environment.yml - internal_tests/ + internal/ manage_exernals/ metplus/ parm/ @@ -253,7 +253,7 @@ developer has Doxygen installed on the host. The Doxygen documentation is useful to contributors and is not necessary for METplus end-users. -The **internal_tests/** directory contains test scripts that are only +The **internal/** directory contains scripts that are only relevant to METplus developers and contributors. The **manage_externals/** directory contains scripts used to diff --git a/internal_tests/scanning/environment/development.docker b/internal/scripts/sonarqube/environment/development.docker similarity index 100% rename from internal_tests/scanning/environment/development.docker rename to internal/scripts/sonarqube/environment/development.docker diff --git a/internal_tests/scanning/environment/development.seneca b/internal/scripts/sonarqube/environment/development.seneca similarity index 100% rename from internal_tests/scanning/environment/development.seneca rename to internal/scripts/sonarqube/environment/development.seneca diff --git a/internal_tests/scanning/sonarqube/run_nightly.sh b/internal/scripts/sonarqube/run_nightly.sh similarity index 94% rename from internal_tests/scanning/sonarqube/run_nightly.sh rename to internal/scripts/sonarqube/run_nightly.sh index 693f962dce..5e41979cc8 100755 --- a/internal_tests/scanning/sonarqube/run_nightly.sh +++ b/internal/scripts/sonarqube/run_nightly.sh @@ -9,7 +9,7 @@ # the following commands: # # git clone https://github.com/dtcenter/METplus -# METplus/internal_tests/scanning/sonarqube/run_nightly.sh name +# METplus/internal/scripts/sonarqube/run_nightly.sh name # # Usage: run_nightly.sh name # where "name" specifies a branch, tag, or hash @@ -39,7 +39,7 @@ SCRIPT_DIR=`dirname $0` if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi # Define the development environment -ENV_FILE=${SCRIPT_DIR}/../environment/development.`hostname` +ENV_FILE=${SCRIPT_DIR}/environment/development.`hostname` if [[ ! -e ${ENV_FILE} ]]; then echo "$0: ERROR -> Development environment file missing: ${ENV_FILE}" exit 1 diff --git a/internal_tests/scanning/sonarqube/run_sonarqube.sh b/internal/scripts/sonarqube/run_sonarqube.sh similarity index 97% rename from internal_tests/scanning/sonarqube/run_sonarqube.sh rename to internal/scripts/sonarqube/run_sonarqube.sh index aae487a928..4b856eab22 100755 --- a/internal_tests/scanning/sonarqube/run_sonarqube.sh +++ b/internal/scripts/sonarqube/run_sonarqube.sh @@ -9,7 +9,7 @@ # then run: # # git clone https://github.com/dtcenter/METplus -# METplus/internal_tests/scanning/sonarqube/run_sonarqube.sh name +# METplus/internal/scripts/sonarqube/run_sonarqube.sh name # # Usage: run_sonarqube.sh name # Test the specified branched version of MET: diff --git a/internal_tests/scanning/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties similarity index 82% rename from internal_tests/scanning/sonarqube/sonar-project.properties rename to internal/scripts/sonarqube/sonar-project.properties index 104f9abca0..677c640bcf 100644 --- a/internal_tests/scanning/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -2,7 +2,7 @@ sonar.projectKey=org.sonarqube:METplus_NB sonar.projectName=METplus Nightly Build sonar.projectVersion=1.0 -sonar.sources=docs,internal_tests,manage_externals,metplus,parm,produtil,ush +sonar.sources=docs,internal,manage_externals,metplus,parm,produtil,ush # The build-wrapper output dir diff --git a/internal_tests/config/GridStatConfig_prob_precip b/internal/tests/config/GridStatConfig_prob_precip similarity index 100% rename from internal_tests/config/GridStatConfig_prob_precip rename to internal/tests/config/GridStatConfig_prob_precip diff --git a/internal_tests/config/STATAnalysisConfig b/internal/tests/config/STATAnalysisConfig similarity index 100% rename from internal_tests/config/STATAnalysisConfig rename to internal/tests/config/STATAnalysisConfig diff --git a/internal_tests/config/ascii2nc_mask_grid.conf b/internal/tests/config/ascii2nc_mask_grid.conf similarity index 100% rename from internal_tests/config/ascii2nc_mask_grid.conf rename to internal/tests/config/ascii2nc_mask_grid.conf diff --git a/internal_tests/config/ascii2nc_mask_poly.conf b/internal/tests/config/ascii2nc_mask_poly.conf similarity index 100% rename from internal_tests/config/ascii2nc_mask_poly.conf rename to internal/tests/config/ascii2nc_mask_poly.conf diff --git a/internal_tests/config/ascii2nc_mask_sid.conf b/internal/tests/config/ascii2nc_mask_sid.conf similarity index 100% rename from internal_tests/config/ascii2nc_mask_sid.conf rename to internal/tests/config/ascii2nc_mask_sid.conf diff --git a/internal_tests/config/ascii2nc_multi_file.conf b/internal/tests/config/ascii2nc_multi_file.conf similarity index 100% rename from internal_tests/config/ascii2nc_multi_file.conf rename to internal/tests/config/ascii2nc_multi_file.conf diff --git a/internal_tests/config/ascii2nc_single_file.conf b/internal/tests/config/ascii2nc_single_file.conf similarity index 100% rename from internal_tests/config/ascii2nc_single_file.conf rename to internal/tests/config/ascii2nc_single_file.conf diff --git a/internal_tests/config/grid_gefs_prob.conf b/internal/tests/config/grid_gefs_prob.conf similarity index 100% rename from internal_tests/config/grid_gefs_prob.conf rename to internal/tests/config/grid_gefs_prob.conf diff --git a/internal_tests/data/accum/20160904/file.2016090412.01h b/internal/tests/data/accum/20160904/file.2016090412.01h similarity index 100% rename from internal_tests/data/accum/20160904/file.2016090412.01h rename to internal/tests/data/accum/20160904/file.2016090412.01h diff --git a/internal_tests/data/accum/20160904/file.2016090413.01h b/internal/tests/data/accum/20160904/file.2016090413.01h similarity index 100% rename from internal_tests/data/accum/20160904/file.2016090413.01h rename to internal/tests/data/accum/20160904/file.2016090413.01h diff --git a/internal_tests/data/accum/20160904/file.2016090414.01h b/internal/tests/data/accum/20160904/file.2016090414.01h similarity index 100% rename from internal_tests/data/accum/20160904/file.2016090414.01h rename to internal/tests/data/accum/20160904/file.2016090414.01h diff --git a/internal_tests/data/accum/20160904/file.2016090415.01h b/internal/tests/data/accum/20160904/file.2016090415.01h similarity index 100% rename from internal_tests/data/accum/20160904/file.2016090415.01h rename to internal/tests/data/accum/20160904/file.2016090415.01h diff --git a/internal_tests/data/accum/20160904/file.2016090416.01h b/internal/tests/data/accum/20160904/file.2016090416.01h similarity index 100% rename from internal_tests/data/accum/20160904/file.2016090416.01h rename to internal/tests/data/accum/20160904/file.2016090416.01h diff --git a/internal_tests/data/accum/20160904/file.2016090417.01h b/internal/tests/data/accum/20160904/file.2016090417.01h similarity index 100% rename from internal_tests/data/accum/20160904/file.2016090417.01h rename to internal/tests/data/accum/20160904/file.2016090417.01h diff --git a/internal_tests/data/accum/20160904/file.2016090418.01h b/internal/tests/data/accum/20160904/file.2016090418.01h similarity index 100% rename from internal_tests/data/accum/20160904/file.2016090418.01h rename to internal/tests/data/accum/20160904/file.2016090418.01h diff --git a/internal_tests/data/accum/20160904/file.2016090418.06h b/internal/tests/data/accum/20160904/file.2016090418.06h similarity index 100% rename from internal_tests/data/accum/20160904/file.2016090418.06h rename to internal/tests/data/accum/20160904/file.2016090418.06h diff --git a/internal_tests/data/config_metplus/both/metplus_config/defaults.conf b/internal/tests/data/config_metplus/both/metplus_config/defaults.conf similarity index 100% rename from internal_tests/data/config_metplus/both/metplus_config/defaults.conf rename to internal/tests/data/config_metplus/both/metplus_config/defaults.conf diff --git a/internal_tests/data/config_metplus/both/metplus_config/metplus_data.conf b/internal/tests/data/config_metplus/both/metplus_config/metplus_data.conf similarity index 100% rename from internal_tests/data/config_metplus/both/metplus_config/metplus_data.conf rename to internal/tests/data/config_metplus/both/metplus_config/metplus_data.conf diff --git a/internal_tests/data/config_metplus/both/metplus_config/metplus_logging.conf b/internal/tests/data/config_metplus/both/metplus_config/metplus_logging.conf similarity index 100% rename from internal_tests/data/config_metplus/both/metplus_config/metplus_logging.conf rename to internal/tests/data/config_metplus/both/metplus_config/metplus_logging.conf diff --git a/internal_tests/data/config_metplus/both/metplus_config/metplus_runtime.conf b/internal/tests/data/config_metplus/both/metplus_config/metplus_runtime.conf similarity index 100% rename from internal_tests/data/config_metplus/both/metplus_config/metplus_runtime.conf rename to internal/tests/data/config_metplus/both/metplus_config/metplus_runtime.conf diff --git a/internal_tests/data/config_metplus/both/metplus_config/metplus_system.conf b/internal/tests/data/config_metplus/both/metplus_config/metplus_system.conf similarity index 100% rename from internal_tests/data/config_metplus/both/metplus_config/metplus_system.conf rename to internal/tests/data/config_metplus/both/metplus_config/metplus_system.conf diff --git a/internal_tests/data/config_metplus/new/metplus_config/defaults.conf b/internal/tests/data/config_metplus/new/metplus_config/defaults.conf similarity index 100% rename from internal_tests/data/config_metplus/new/metplus_config/defaults.conf rename to internal/tests/data/config_metplus/new/metplus_config/defaults.conf diff --git a/internal_tests/data/config_metplus/old/metplus_config/metplus_data.conf b/internal/tests/data/config_metplus/old/metplus_config/metplus_data.conf similarity index 100% rename from internal_tests/data/config_metplus/old/metplus_config/metplus_data.conf rename to internal/tests/data/config_metplus/old/metplus_config/metplus_data.conf diff --git a/internal_tests/data/config_metplus/old/metplus_config/metplus_logging.conf b/internal/tests/data/config_metplus/old/metplus_config/metplus_logging.conf similarity index 100% rename from internal_tests/data/config_metplus/old/metplus_config/metplus_logging.conf rename to internal/tests/data/config_metplus/old/metplus_config/metplus_logging.conf diff --git a/internal_tests/data/config_metplus/old/metplus_config/metplus_runtime.conf b/internal/tests/data/config_metplus/old/metplus_config/metplus_runtime.conf similarity index 100% rename from internal_tests/data/config_metplus/old/metplus_config/metplus_runtime.conf rename to internal/tests/data/config_metplus/old/metplus_config/metplus_runtime.conf diff --git a/internal_tests/data/config_metplus/old/metplus_config/metplus_system.conf b/internal/tests/data/config_metplus/old/metplus_config/metplus_system.conf similarity index 100% rename from internal_tests/data/config_metplus/old/metplus_config/metplus_system.conf rename to internal/tests/data/config_metplus/old/metplus_config/metplus_system.conf diff --git a/internal_tests/data/daily/file.20180131.txt b/internal/tests/data/daily/file.20180131.txt similarity index 100% rename from internal_tests/data/daily/file.20180131.txt rename to internal/tests/data/daily/file.20180131.txt diff --git a/internal_tests/data/daily/file.20180201.txt b/internal/tests/data/daily/file.20180201.txt similarity index 100% rename from internal_tests/data/daily/file.20180201.txt rename to internal/tests/data/daily/file.20180201.txt diff --git a/internal_tests/data/daily/file.20180202.txt b/internal/tests/data/daily/file.20180202.txt similarity index 100% rename from internal_tests/data/daily/file.20180202.txt rename to internal/tests/data/daily/file.20180202.txt diff --git a/internal_tests/data/daily/file.20180207.txt b/internal/tests/data/daily/file.20180207.txt similarity index 100% rename from internal_tests/data/daily/file.20180207.txt rename to internal/tests/data/daily/file.20180207.txt diff --git a/internal_tests/data/ens/2009123112/arw-fer-gep1/d01_2009123112_02400.grib b/internal/tests/data/ens/2009123112/arw-fer-gep1/d01_2009123112_02400.grib similarity index 100% rename from internal_tests/data/ens/2009123112/arw-fer-gep1/d01_2009123112_02400.grib rename to internal/tests/data/ens/2009123112/arw-fer-gep1/d01_2009123112_02400.grib diff --git a/internal_tests/data/ens/2009123112/arw-fer-gep1/d01_2009123112_04800.grib b/internal/tests/data/ens/2009123112/arw-fer-gep1/d01_2009123112_04800.grib similarity index 100% rename from internal_tests/data/ens/2009123112/arw-fer-gep1/d01_2009123112_04800.grib rename to internal/tests/data/ens/2009123112/arw-fer-gep1/d01_2009123112_04800.grib diff --git a/internal_tests/data/ens/2009123112/arw-fer-gep5/d01_2009123112_02400.grib b/internal/tests/data/ens/2009123112/arw-fer-gep5/d01_2009123112_02400.grib similarity index 100% rename from internal_tests/data/ens/2009123112/arw-fer-gep5/d01_2009123112_02400.grib rename to internal/tests/data/ens/2009123112/arw-fer-gep5/d01_2009123112_02400.grib diff --git a/internal_tests/data/ens/2009123112/arw-fer-gep5/d01_2009123112_04800.grib b/internal/tests/data/ens/2009123112/arw-fer-gep5/d01_2009123112_04800.grib similarity index 100% rename from internal_tests/data/ens/2009123112/arw-fer-gep5/d01_2009123112_04800.grib rename to internal/tests/data/ens/2009123112/arw-fer-gep5/d01_2009123112_04800.grib diff --git a/internal_tests/data/ens/2009123112/arw-sch-gep2/d01_2009123112_02400.grib b/internal/tests/data/ens/2009123112/arw-sch-gep2/d01_2009123112_02400.grib similarity index 100% rename from internal_tests/data/ens/2009123112/arw-sch-gep2/d01_2009123112_02400.grib rename to internal/tests/data/ens/2009123112/arw-sch-gep2/d01_2009123112_02400.grib diff --git a/internal_tests/data/ens/2009123112/arw-sch-gep2/d01_2009123112_04800.grib b/internal/tests/data/ens/2009123112/arw-sch-gep2/d01_2009123112_04800.grib similarity index 100% rename from internal_tests/data/ens/2009123112/arw-sch-gep2/d01_2009123112_04800.grib rename to internal/tests/data/ens/2009123112/arw-sch-gep2/d01_2009123112_04800.grib diff --git a/internal_tests/data/ens/2009123112/arw-sch-gep6/d01_2009123112_02400.grib b/internal/tests/data/ens/2009123112/arw-sch-gep6/d01_2009123112_02400.grib similarity index 100% rename from internal_tests/data/ens/2009123112/arw-sch-gep6/d01_2009123112_02400.grib rename to internal/tests/data/ens/2009123112/arw-sch-gep6/d01_2009123112_02400.grib diff --git a/internal_tests/data/ens/2009123112/arw-sch-gep6/d01_2009123112_04800.grib b/internal/tests/data/ens/2009123112/arw-sch-gep6/d01_2009123112_04800.grib similarity index 100% rename from internal_tests/data/ens/2009123112/arw-sch-gep6/d01_2009123112_04800.grib rename to internal/tests/data/ens/2009123112/arw-sch-gep6/d01_2009123112_04800.grib diff --git a/internal_tests/data/ens/2009123112/arw-tom-gep3/d01_2009123112_02400.grib b/internal/tests/data/ens/2009123112/arw-tom-gep3/d01_2009123112_02400.grib similarity index 100% rename from internal_tests/data/ens/2009123112/arw-tom-gep3/d01_2009123112_02400.grib rename to internal/tests/data/ens/2009123112/arw-tom-gep3/d01_2009123112_02400.grib diff --git a/internal_tests/data/ens/2009123112/arw-tom-gep3/d01_2009123112_04800.grib b/internal/tests/data/ens/2009123112/arw-tom-gep3/d01_2009123112_04800.grib similarity index 100% rename from internal_tests/data/ens/2009123112/arw-tom-gep3/d01_2009123112_04800.grib rename to internal/tests/data/ens/2009123112/arw-tom-gep3/d01_2009123112_04800.grib diff --git a/internal_tests/data/ens/2009123112/arw-tom-gep7/d01_2009123112_02400.grib b/internal/tests/data/ens/2009123112/arw-tom-gep7/d01_2009123112_02400.grib similarity index 100% rename from internal_tests/data/ens/2009123112/arw-tom-gep7/d01_2009123112_02400.grib rename to internal/tests/data/ens/2009123112/arw-tom-gep7/d01_2009123112_02400.grib diff --git a/internal_tests/data/ens/2009123112/arw-tom-gep7/d01_2009123112_04800.grib b/internal/tests/data/ens/2009123112/arw-tom-gep7/d01_2009123112_04800.grib similarity index 100% rename from internal_tests/data/ens/2009123112/arw-tom-gep7/d01_2009123112_04800.grib rename to internal/tests/data/ens/2009123112/arw-tom-gep7/d01_2009123112_04800.grib diff --git a/internal_tests/data/ens/2009123118/arw-fer-gep1/d01_2009123118_02400.grib b/internal/tests/data/ens/2009123118/arw-fer-gep1/d01_2009123118_02400.grib similarity index 100% rename from internal_tests/data/ens/2009123118/arw-fer-gep1/d01_2009123118_02400.grib rename to internal/tests/data/ens/2009123118/arw-fer-gep1/d01_2009123118_02400.grib diff --git a/internal_tests/data/ens/2009123118/arw-fer-gep1/d01_2009123118_04800.grib b/internal/tests/data/ens/2009123118/arw-fer-gep1/d01_2009123118_04800.grib similarity index 100% rename from internal_tests/data/ens/2009123118/arw-fer-gep1/d01_2009123118_04800.grib rename to internal/tests/data/ens/2009123118/arw-fer-gep1/d01_2009123118_04800.grib diff --git a/internal_tests/data/ens/2009123118/arw-fer-gep5/d01_2009123118_02400.grib b/internal/tests/data/ens/2009123118/arw-fer-gep5/d01_2009123118_02400.grib similarity index 100% rename from internal_tests/data/ens/2009123118/arw-fer-gep5/d01_2009123118_02400.grib rename to internal/tests/data/ens/2009123118/arw-fer-gep5/d01_2009123118_02400.grib diff --git a/internal_tests/data/ens/2009123118/arw-fer-gep5/d01_2009123118_04800.grib b/internal/tests/data/ens/2009123118/arw-fer-gep5/d01_2009123118_04800.grib similarity index 100% rename from internal_tests/data/ens/2009123118/arw-fer-gep5/d01_2009123118_04800.grib rename to internal/tests/data/ens/2009123118/arw-fer-gep5/d01_2009123118_04800.grib diff --git a/internal_tests/data/ens/2009123118/arw-sch-gep2/d01_2009123118_02400.grib b/internal/tests/data/ens/2009123118/arw-sch-gep2/d01_2009123118_02400.grib similarity index 100% rename from internal_tests/data/ens/2009123118/arw-sch-gep2/d01_2009123118_02400.grib rename to internal/tests/data/ens/2009123118/arw-sch-gep2/d01_2009123118_02400.grib diff --git a/internal_tests/data/ens/2009123118/arw-sch-gep2/d01_2009123118_04800.grib b/internal/tests/data/ens/2009123118/arw-sch-gep2/d01_2009123118_04800.grib similarity index 100% rename from internal_tests/data/ens/2009123118/arw-sch-gep2/d01_2009123118_04800.grib rename to internal/tests/data/ens/2009123118/arw-sch-gep2/d01_2009123118_04800.grib diff --git a/internal_tests/data/ens/2009123118/arw-sch-gep6/d01_2009123118_02400.grib b/internal/tests/data/ens/2009123118/arw-sch-gep6/d01_2009123118_02400.grib similarity index 100% rename from internal_tests/data/ens/2009123118/arw-sch-gep6/d01_2009123118_02400.grib rename to internal/tests/data/ens/2009123118/arw-sch-gep6/d01_2009123118_02400.grib diff --git a/internal_tests/data/ens/2009123118/arw-sch-gep6/d01_2009123118_04800.grib b/internal/tests/data/ens/2009123118/arw-sch-gep6/d01_2009123118_04800.grib similarity index 100% rename from internal_tests/data/ens/2009123118/arw-sch-gep6/d01_2009123118_04800.grib rename to internal/tests/data/ens/2009123118/arw-sch-gep6/d01_2009123118_04800.grib diff --git a/internal_tests/data/ens/2009123118/arw-tom-gep3/d01_2009123118_02400.grib b/internal/tests/data/ens/2009123118/arw-tom-gep3/d01_2009123118_02400.grib similarity index 100% rename from internal_tests/data/ens/2009123118/arw-tom-gep3/d01_2009123118_02400.grib rename to internal/tests/data/ens/2009123118/arw-tom-gep3/d01_2009123118_02400.grib diff --git a/internal_tests/data/ens/2009123118/arw-tom-gep3/d01_2009123118_04800.grib b/internal/tests/data/ens/2009123118/arw-tom-gep3/d01_2009123118_04800.grib similarity index 100% rename from internal_tests/data/ens/2009123118/arw-tom-gep3/d01_2009123118_04800.grib rename to internal/tests/data/ens/2009123118/arw-tom-gep3/d01_2009123118_04800.grib diff --git a/internal_tests/data/ens/2009123118/arw-tom-gep7/d01_2009123118_02400.grib b/internal/tests/data/ens/2009123118/arw-tom-gep7/d01_2009123118_02400.grib similarity index 100% rename from internal_tests/data/ens/2009123118/arw-tom-gep7/d01_2009123118_02400.grib rename to internal/tests/data/ens/2009123118/arw-tom-gep7/d01_2009123118_02400.grib diff --git a/internal_tests/data/ens/2009123118/arw-tom-gep7/d01_2009123118_04800.grib b/internal/tests/data/ens/2009123118/arw-tom-gep7/d01_2009123118_04800.grib similarity index 100% rename from internal_tests/data/ens/2009123118/arw-tom-gep7/d01_2009123118_04800.grib rename to internal/tests/data/ens/2009123118/arw-tom-gep7/d01_2009123118_04800.grib diff --git a/internal_tests/data/fake/20180201/data_2018020100.nc b/internal/tests/data/fake/20180201/data_2018020100.nc similarity index 100% rename from internal_tests/data/fake/20180201/data_2018020100.nc rename to internal/tests/data/fake/20180201/data_2018020100.nc diff --git a/internal_tests/data/fake/20180201/data_2018020106.nc b/internal/tests/data/fake/20180201/data_2018020106.nc similarity index 100% rename from internal_tests/data/fake/20180201/data_2018020106.nc rename to internal/tests/data/fake/20180201/data_2018020106.nc diff --git a/internal_tests/data/fake/20180201/data_2018020112.nc b/internal/tests/data/fake/20180201/data_2018020112.nc similarity index 100% rename from internal_tests/data/fake/20180201/data_2018020112.nc rename to internal/tests/data/fake/20180201/data_2018020112.nc diff --git a/internal_tests/data/fake/20180201/data_2018020118.nc b/internal/tests/data/fake/20180201/data_2018020118.nc similarity index 100% rename from internal_tests/data/fake/20180201/data_2018020118.nc rename to internal/tests/data/fake/20180201/data_2018020118.nc diff --git a/internal_tests/data/fcst/20160904/file.2016090415f003.nc b/internal/tests/data/fcst/20160904/file.2016090415f003.nc similarity index 100% rename from internal_tests/data/fcst/20160904/file.2016090415f003.nc rename to internal/tests/data/fcst/20160904/file.2016090415f003.nc diff --git a/internal_tests/data/fcst/20160904/file.2016090415f009.nc b/internal/tests/data/fcst/20160904/file.2016090415f009.nc similarity index 100% rename from internal_tests/data/fcst/20160904/file.2016090415f009.nc rename to internal/tests/data/fcst/20160904/file.2016090415f009.nc diff --git a/internal_tests/data/fcst/20170510/20170510_i00_f003_HRRRTLE_PHPT.grb2 b/internal/tests/data/fcst/20170510/20170510_i00_f003_HRRRTLE_PHPT.grb2 similarity index 100% rename from internal_tests/data/fcst/20170510/20170510_i00_f003_HRRRTLE_PHPT.grb2 rename to internal/tests/data/fcst/20170510/20170510_i00_f003_HRRRTLE_PHPT.grb2 diff --git a/internal_tests/data/fcst/20170510/20170510_i01_f002_HRRRTLE_PHPT.grb2 b/internal/tests/data/fcst/20170510/20170510_i01_f002_HRRRTLE_PHPT.grb2 similarity index 100% rename from internal_tests/data/fcst/20170510/20170510_i01_f002_HRRRTLE_PHPT.grb2 rename to internal/tests/data/fcst/20170510/20170510_i01_f002_HRRRTLE_PHPT.grb2 diff --git a/internal_tests/data/fcst/20170510/20170510_i02_f001_HRRRTLE_PHPT.grb2 b/internal/tests/data/fcst/20170510/20170510_i02_f001_HRRRTLE_PHPT.grb2 similarity index 100% rename from internal_tests/data/fcst/20170510/20170510_i02_f001_HRRRTLE_PHPT.grb2 rename to internal/tests/data/fcst/20170510/20170510_i02_f001_HRRRTLE_PHPT.grb2 diff --git a/internal_tests/data/fcst/20170510/20170510_i03_f001_HRRRTLE_PHPT.grb2 b/internal/tests/data/fcst/20170510/20170510_i03_f001_HRRRTLE_PHPT.grb2 similarity index 100% rename from internal_tests/data/fcst/20170510/20170510_i03_f001_HRRRTLE_PHPT.grb2 rename to internal/tests/data/fcst/20170510/20170510_i03_f001_HRRRTLE_PHPT.grb2 diff --git a/internal_tests/data/fcst/20170510/20170510_i03_f002_HRRRTLE_PHPT.grb2 b/internal/tests/data/fcst/20170510/20170510_i03_f002_HRRRTLE_PHPT.grb2 similarity index 100% rename from internal_tests/data/fcst/20170510/20170510_i03_f002_HRRRTLE_PHPT.grb2 rename to internal/tests/data/fcst/20170510/20170510_i03_f002_HRRRTLE_PHPT.grb2 diff --git a/internal_tests/data/fcst/20170510/20170510_i03_f003_HRRRTLE_PHPT.grb2 b/internal/tests/data/fcst/20170510/20170510_i03_f003_HRRRTLE_PHPT.grb2 similarity index 100% rename from internal_tests/data/fcst/20170510/20170510_i03_f003_HRRRTLE_PHPT.grb2 rename to internal/tests/data/fcst/20170510/20170510_i03_f003_HRRRTLE_PHPT.grb2 diff --git a/internal_tests/data/fcst/20170510/20170510_i03_f006_HRRRTLE_PHPT.grb2 b/internal/tests/data/fcst/20170510/20170510_i03_f006_HRRRTLE_PHPT.grb2 similarity index 100% rename from internal_tests/data/fcst/20170510/20170510_i03_f006_HRRRTLE_PHPT.grb2 rename to internal/tests/data/fcst/20170510/20170510_i03_f006_HRRRTLE_PHPT.grb2 diff --git a/internal_tests/data/fcst/20170510/20170510_i03_f012_HRRRTLE_PHPT.grb2 b/internal/tests/data/fcst/20170510/20170510_i03_f012_HRRRTLE_PHPT.grb2 similarity index 100% rename from internal_tests/data/fcst/20170510/20170510_i03_f012_HRRRTLE_PHPT.grb2 rename to internal/tests/data/fcst/20170510/20170510_i03_f012_HRRRTLE_PHPT.grb2 diff --git a/internal_tests/data/fcst/20180201/file.2018020112f009.nc b/internal/tests/data/fcst/20180201/file.2018020112f009.nc similarity index 100% rename from internal_tests/data/fcst/20180201/file.2018020112f009.nc rename to internal/tests/data/fcst/20180201/file.2018020112f009.nc diff --git a/internal_tests/data/fcst/20180201/file.2018020115f006.nc b/internal/tests/data/fcst/20180201/file.2018020115f006.nc similarity index 100% rename from internal_tests/data/fcst/20180201/file.2018020115f006.nc rename to internal/tests/data/fcst/20180201/file.2018020115f006.nc diff --git a/internal_tests/data/fcst/20180201/file.2018020118f003.nc b/internal/tests/data/fcst/20180201/file.2018020118f003.nc similarity index 100% rename from internal_tests/data/fcst/20180201/file.2018020118f003.nc rename to internal/tests/data/fcst/20180201/file.2018020118f003.nc diff --git a/internal_tests/data/fcst/20180201/file.2018020119f003.nc b/internal/tests/data/fcst/20180201/file.2018020119f003.nc similarity index 100% rename from internal_tests/data/fcst/20180201/file.2018020119f003.nc rename to internal/tests/data/fcst/20180201/file.2018020119f003.nc diff --git a/internal_tests/data/fcst/file.2018013118f012.nc b/internal/tests/data/fcst/file.2018013118f012.nc similarity index 100% rename from internal_tests/data/fcst/file.2018013118f012.nc rename to internal/tests/data/fcst/file.2018013118f012.nc diff --git a/internal_tests/data/fcst/file.2018020115f006.nc b/internal/tests/data/fcst/file.2018020115f006.nc similarity index 100% rename from internal_tests/data/fcst/file.2018020115f006.nc rename to internal/tests/data/fcst/file.2018020115f006.nc diff --git a/internal_tests/data/fcst/file.2018020118f003.nc b/internal/tests/data/fcst/file.2018020118f003.nc similarity index 100% rename from internal_tests/data/fcst/file.2018020118f003.nc rename to internal/tests/data/fcst/file.2018020118f003.nc diff --git a/internal_tests/data/fcst/file.2018020119f003.nc b/internal/tests/data/fcst/file.2018020119f003.nc similarity index 100% rename from internal_tests/data/fcst/file.2018020119f003.nc rename to internal/tests/data/fcst/file.2018020119f003.nc diff --git a/internal_tests/data/file_lists/FCST_FILES_ML1200942014 b/internal/tests/data/file_lists/FCST_FILES_ML1200942014 similarity index 100% rename from internal_tests/data/file_lists/FCST_FILES_ML1200942014 rename to internal/tests/data/file_lists/FCST_FILES_ML1200942014 diff --git a/internal_tests/data/mtd/fake_mtd_2d.txt b/internal/tests/data/mtd/fake_mtd_2d.txt similarity index 100% rename from internal_tests/data/mtd/fake_mtd_2d.txt rename to internal/tests/data/mtd/fake_mtd_2d.txt diff --git a/internal_tests/data/obs/14z.prepbufr.tm02.20200201 b/internal/tests/data/obs/14z.prepbufr.tm02.20200201 similarity index 100% rename from internal_tests/data/obs/14z.prepbufr.tm02.20200201 rename to internal/tests/data/obs/14z.prepbufr.tm02.20200201 diff --git a/internal_tests/data/obs/18z.prepbufr.tm06.20200201 b/internal/tests/data/obs/18z.prepbufr.tm06.20200201 similarity index 100% rename from internal_tests/data/obs/18z.prepbufr.tm06.20200201 rename to internal/tests/data/obs/18z.prepbufr.tm06.20200201 diff --git a/internal_tests/data/obs/20170510/qpe_2017051003_A06.nc b/internal/tests/data/obs/20170510/qpe_2017051003_A06.nc similarity index 100% rename from internal_tests/data/obs/20170510/qpe_2017051003_A06.nc rename to internal/tests/data/obs/20170510/qpe_2017051003_A06.nc diff --git a/internal_tests/data/obs/20170510/qpe_2017051004_A06.nc b/internal/tests/data/obs/20170510/qpe_2017051004_A06.nc similarity index 100% rename from internal_tests/data/obs/20170510/qpe_2017051004_A06.nc rename to internal/tests/data/obs/20170510/qpe_2017051004_A06.nc diff --git a/internal_tests/data/obs/20170510/qpe_2017051005_A06.nc b/internal/tests/data/obs/20170510/qpe_2017051005_A06.nc similarity index 100% rename from internal_tests/data/obs/20170510/qpe_2017051005_A06.nc rename to internal/tests/data/obs/20170510/qpe_2017051005_A06.nc diff --git a/internal_tests/data/obs/20170510/qpe_2017051006_A06.nc b/internal/tests/data/obs/20170510/qpe_2017051006_A06.nc similarity index 100% rename from internal_tests/data/obs/20170510/qpe_2017051006_A06.nc rename to internal/tests/data/obs/20170510/qpe_2017051006_A06.nc diff --git a/internal_tests/data/obs/20170510/qpe_2017051009_A06.nc b/internal/tests/data/obs/20170510/qpe_2017051009_A06.nc similarity index 100% rename from internal_tests/data/obs/20170510/qpe_2017051009_A06.nc rename to internal/tests/data/obs/20170510/qpe_2017051009_A06.nc diff --git a/internal_tests/data/obs/20170510/qpe_2017051012_A06.nc b/internal/tests/data/obs/20170510/qpe_2017051012_A06.nc similarity index 100% rename from internal_tests/data/obs/20170510/qpe_2017051012_A06.nc rename to internal/tests/data/obs/20170510/qpe_2017051012_A06.nc diff --git a/internal_tests/data/obs/20170510/qpe_2017051015_A06.nc b/internal/tests/data/obs/20170510/qpe_2017051015_A06.nc similarity index 100% rename from internal_tests/data/obs/20170510/qpe_2017051015_A06.nc rename to internal/tests/data/obs/20170510/qpe_2017051015_A06.nc diff --git a/internal_tests/data/obs/20180131/20180131_2345 b/internal/tests/data/obs/20180131/20180131_2345 similarity index 100% rename from internal_tests/data/obs/20180131/20180131_2345 rename to internal/tests/data/obs/20180131/20180131_2345 diff --git a/internal_tests/data/obs/20180201/20180201_0013 b/internal/tests/data/obs/20180201/20180201_0013 similarity index 100% rename from internal_tests/data/obs/20180201/20180201_0013 rename to internal/tests/data/obs/20180201/20180201_0013 diff --git a/internal_tests/data/obs/20180201_0045 b/internal/tests/data/obs/20180201_0045 similarity index 100% rename from internal_tests/data/obs/20180201_0045 rename to internal/tests/data/obs/20180201_0045 diff --git a/internal_tests/data/obs/20180202/20180202_0013 b/internal/tests/data/obs/20180202/20180202_0013 similarity index 100% rename from internal_tests/data/obs/20180202/20180202_0013 rename to internal/tests/data/obs/20180202/20180202_0013 diff --git a/internal_tests/data/pcp_in/add/20190802_i1800_m0_f1805.nc b/internal/tests/data/pcp_in/add/20190802_i1800_m0_f1805.nc similarity index 100% rename from internal_tests/data/pcp_in/add/20190802_i1800_m0_f1805.nc rename to internal/tests/data/pcp_in/add/20190802_i1800_m0_f1805.nc diff --git a/internal_tests/data/pcp_in/add/20190802_i1800_m0_f1810.nc b/internal/tests/data/pcp_in/add/20190802_i1800_m0_f1810.nc similarity index 100% rename from internal_tests/data/pcp_in/add/20190802_i1800_m0_f1810.nc rename to internal/tests/data/pcp_in/add/20190802_i1800_m0_f1810.nc diff --git a/internal_tests/data/pcp_in/add/20190802_i1800_m0_f1815.nc b/internal/tests/data/pcp_in/add/20190802_i1800_m0_f1815.nc similarity index 100% rename from internal_tests/data/pcp_in/add/20190802_i1800_m0_f1815.nc rename to internal/tests/data/pcp_in/add/20190802_i1800_m0_f1815.nc diff --git a/internal_tests/data/pcp_in/bucket/2012040900_F006.grib b/internal/tests/data/pcp_in/bucket/2012040900_F006.grib similarity index 100% rename from internal_tests/data/pcp_in/bucket/2012040900_F006.grib rename to internal/tests/data/pcp_in/bucket/2012040900_F006.grib diff --git a/internal_tests/data/pcp_in/bucket/2012040900_F012.grib b/internal/tests/data/pcp_in/bucket/2012040900_F012.grib similarity index 100% rename from internal_tests/data/pcp_in/bucket/2012040900_F012.grib rename to internal/tests/data/pcp_in/bucket/2012040900_F012.grib diff --git a/internal_tests/data/pcp_in/bucket/2012040900_F015.grib b/internal/tests/data/pcp_in/bucket/2012040900_F015.grib similarity index 100% rename from internal_tests/data/pcp_in/bucket/2012040900_F015.grib rename to internal/tests/data/pcp_in/bucket/2012040900_F015.grib diff --git a/internal_tests/data/pcp_in/derive/2005080700/00.tm00_G212 b/internal/tests/data/pcp_in/derive/2005080700/00.tm00_G212 similarity index 100% rename from internal_tests/data/pcp_in/derive/2005080700/00.tm00_G212 rename to internal/tests/data/pcp_in/derive/2005080700/00.tm00_G212 diff --git a/internal_tests/data/pcp_in/derive/2005080700/03.tm00_G212 b/internal/tests/data/pcp_in/derive/2005080700/03.tm00_G212 similarity index 100% rename from internal_tests/data/pcp_in/derive/2005080700/03.tm00_G212 rename to internal/tests/data/pcp_in/derive/2005080700/03.tm00_G212 diff --git a/internal_tests/data/pcp_in/derive/2005080700/06.tm00_G212 b/internal/tests/data/pcp_in/derive/2005080700/06.tm00_G212 similarity index 100% rename from internal_tests/data/pcp_in/derive/2005080700/06.tm00_G212 rename to internal/tests/data/pcp_in/derive/2005080700/06.tm00_G212 diff --git a/internal_tests/data/pcp_in/derive/2005080700/09.tm00_G212 b/internal/tests/data/pcp_in/derive/2005080700/09.tm00_G212 similarity index 100% rename from internal_tests/data/pcp_in/derive/2005080700/09.tm00_G212 rename to internal/tests/data/pcp_in/derive/2005080700/09.tm00_G212 diff --git a/internal_tests/data/pcp_in/derive/2005080700/12.tm00_G212 b/internal/tests/data/pcp_in/derive/2005080700/12.tm00_G212 similarity index 100% rename from internal_tests/data/pcp_in/derive/2005080700/12.tm00_G212 rename to internal/tests/data/pcp_in/derive/2005080700/12.tm00_G212 diff --git a/internal_tests/data/pcp_in/derive/2005080700/15.tm00_G212 b/internal/tests/data/pcp_in/derive/2005080700/15.tm00_G212 similarity index 100% rename from internal_tests/data/pcp_in/derive/2005080700/15.tm00_G212 rename to internal/tests/data/pcp_in/derive/2005080700/15.tm00_G212 diff --git a/internal_tests/data/pcp_in/derive/2005080700/18.tm00_G212 b/internal/tests/data/pcp_in/derive/2005080700/18.tm00_G212 similarity index 100% rename from internal_tests/data/pcp_in/derive/2005080700/18.tm00_G212 rename to internal/tests/data/pcp_in/derive/2005080700/18.tm00_G212 diff --git a/internal_tests/data/pcp_in/derive/2005080700/21.tm00_G212 b/internal/tests/data/pcp_in/derive/2005080700/21.tm00_G212 similarity index 100% rename from internal_tests/data/pcp_in/derive/2005080700/21.tm00_G212 rename to internal/tests/data/pcp_in/derive/2005080700/21.tm00_G212 diff --git a/internal_tests/data/pcp_in/derive/2005080700/24.tm00_G212 b/internal/tests/data/pcp_in/derive/2005080700/24.tm00_G212 similarity index 100% rename from internal_tests/data/pcp_in/derive/2005080700/24.tm00_G212 rename to internal/tests/data/pcp_in/derive/2005080700/24.tm00_G212 diff --git a/internal_tests/data/pcp_in/loop_custom/ens1/2009123112_02400.grib b/internal/tests/data/pcp_in/loop_custom/ens1/2009123112_02400.grib similarity index 100% rename from internal_tests/data/pcp_in/loop_custom/ens1/2009123112_02400.grib rename to internal/tests/data/pcp_in/loop_custom/ens1/2009123112_02400.grib diff --git a/internal_tests/data/pcp_in/loop_custom/ens2/2009123112_02400.grib b/internal/tests/data/pcp_in/loop_custom/ens2/2009123112_02400.grib similarity index 100% rename from internal_tests/data/pcp_in/loop_custom/ens2/2009123112_02400.grib rename to internal/tests/data/pcp_in/loop_custom/ens2/2009123112_02400.grib diff --git a/internal_tests/data/pcp_in/loop_custom/ens3/2009123112_02400.grib b/internal/tests/data/pcp_in/loop_custom/ens3/2009123112_02400.grib similarity index 100% rename from internal_tests/data/pcp_in/loop_custom/ens3/2009123112_02400.grib rename to internal/tests/data/pcp_in/loop_custom/ens3/2009123112_02400.grib diff --git a/internal_tests/data/pcp_in/loop_custom/ens4/2009123112_02400.grib b/internal/tests/data/pcp_in/loop_custom/ens4/2009123112_02400.grib similarity index 100% rename from internal_tests/data/pcp_in/loop_custom/ens4/2009123112_02400.grib rename to internal/tests/data/pcp_in/loop_custom/ens4/2009123112_02400.grib diff --git a/internal_tests/data/pcp_in/loop_custom/ens5/2009123112_02400.grib b/internal/tests/data/pcp_in/loop_custom/ens5/2009123112_02400.grib similarity index 100% rename from internal_tests/data/pcp_in/loop_custom/ens5/2009123112_02400.grib rename to internal/tests/data/pcp_in/loop_custom/ens5/2009123112_02400.grib diff --git a/internal_tests/data/pcp_in/loop_custom/ens6/2009123112_02400.grib b/internal/tests/data/pcp_in/loop_custom/ens6/2009123112_02400.grib similarity index 100% rename from internal_tests/data/pcp_in/loop_custom/ens6/2009123112_02400.grib rename to internal/tests/data/pcp_in/loop_custom/ens6/2009123112_02400.grib diff --git a/internal_tests/data/stat_data/another_fake_filter_20141214_00.tcst b/internal/tests/data/stat_data/another_fake_filter_20141214_00.tcst similarity index 100% rename from internal_tests/data/stat_data/another_fake_filter_20141214_00.tcst rename to internal/tests/data/stat_data/another_fake_filter_20141214_00.tcst diff --git a/internal_tests/data/stat_data/empty_filter.tcst b/internal/tests/data/stat_data/empty_filter.tcst similarity index 100% rename from internal_tests/data/stat_data/empty_filter.tcst rename to internal/tests/data/stat_data/empty_filter.tcst diff --git a/internal_tests/data/stat_data/fake_filter_20141214_00.tcst b/internal/tests/data/stat_data/fake_filter_20141214_00.tcst similarity index 100% rename from internal_tests/data/stat_data/fake_filter_20141214_00.tcst rename to internal/tests/data/stat_data/fake_filter_20141214_00.tcst diff --git a/internal_tests/data/stat_data/test_20190101.stat b/internal/tests/data/stat_data/test_20190101.stat similarity index 100% rename from internal_tests/data/stat_data/test_20190101.stat rename to internal/tests/data/stat_data/test_20190101.stat diff --git a/internal_tests/data/tc_gen/edeck/edeck_fake_2016020100 b/internal/tests/data/tc_gen/edeck/edeck_fake_2016020100 similarity index 100% rename from internal_tests/data/tc_gen/edeck/edeck_fake_2016020100 rename to internal/tests/data/tc_gen/edeck/edeck_fake_2016020100 diff --git a/internal_tests/data/tc_gen/edeck/edeck_fake_2016020112 b/internal/tests/data/tc_gen/edeck/edeck_fake_2016020112 similarity index 100% rename from internal_tests/data/tc_gen/edeck/edeck_fake_2016020112 rename to internal/tests/data/tc_gen/edeck/edeck_fake_2016020112 diff --git a/internal_tests/data/tc_gen/edeck/edeck_fake_2016020200 b/internal/tests/data/tc_gen/edeck/edeck_fake_2016020200 similarity index 100% rename from internal_tests/data/tc_gen/edeck/edeck_fake_2016020200 rename to internal/tests/data/tc_gen/edeck/edeck_fake_2016020200 diff --git a/internal_tests/data/tc_gen/edeck/edeck_fake_2016020212 b/internal/tests/data/tc_gen/edeck/edeck_fake_2016020212 similarity index 100% rename from internal_tests/data/tc_gen/edeck/edeck_fake_2016020212 rename to internal/tests/data/tc_gen/edeck/edeck_fake_2016020212 diff --git a/internal_tests/data/tc_gen/edeck/edeck_fake_2016103100 b/internal/tests/data/tc_gen/edeck/edeck_fake_2016103100 similarity index 100% rename from internal_tests/data/tc_gen/edeck/edeck_fake_2016103100 rename to internal/tests/data/tc_gen/edeck/edeck_fake_2016103100 diff --git a/internal_tests/data/tc_gen/edeck/edeck_fake_2017020100 b/internal/tests/data/tc_gen/edeck/edeck_fake_2017020100 similarity index 100% rename from internal_tests/data/tc_gen/edeck/edeck_fake_2017020100 rename to internal/tests/data/tc_gen/edeck/edeck_fake_2017020100 diff --git a/internal_tests/data/tc_gen/edeck/edeck_fake_2017020112 b/internal/tests/data/tc_gen/edeck/edeck_fake_2017020112 similarity index 100% rename from internal_tests/data/tc_gen/edeck/edeck_fake_2017020112 rename to internal/tests/data/tc_gen/edeck/edeck_fake_2017020112 diff --git a/internal_tests/data/tc_gen/edeck/edeck_fake_2017020200 b/internal/tests/data/tc_gen/edeck/edeck_fake_2017020200 similarity index 100% rename from internal_tests/data/tc_gen/edeck/edeck_fake_2017020200 rename to internal/tests/data/tc_gen/edeck/edeck_fake_2017020200 diff --git a/internal_tests/data/tc_gen/edeck/edeck_fake_2017020212 b/internal/tests/data/tc_gen/edeck/edeck_fake_2017020212 similarity index 100% rename from internal_tests/data/tc_gen/edeck/edeck_fake_2017020212 rename to internal/tests/data/tc_gen/edeck/edeck_fake_2017020212 diff --git a/internal_tests/data/tc_gen/edeck/edeck_fake_2017103100 b/internal/tests/data/tc_gen/edeck/edeck_fake_2017103100 similarity index 100% rename from internal_tests/data/tc_gen/edeck/edeck_fake_2017103100 rename to internal/tests/data/tc_gen/edeck/edeck_fake_2017103100 diff --git a/internal_tests/data/tc_gen/edeck/edeck_fake_2017103112 b/internal/tests/data/tc_gen/edeck/edeck_fake_2017103112 similarity index 100% rename from internal_tests/data/tc_gen/edeck/edeck_fake_2017103112 rename to internal/tests/data/tc_gen/edeck/edeck_fake_2017103112 diff --git a/internal_tests/data/tc_gen/edeck/edeck_fake_2017123018 b/internal/tests/data/tc_gen/edeck/edeck_fake_2017123018 similarity index 100% rename from internal_tests/data/tc_gen/edeck/edeck_fake_2017123018 rename to internal/tests/data/tc_gen/edeck/edeck_fake_2017123018 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2016020100 b/internal/tests/data/tc_gen/genesis/genesis_fake_2016020100 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2016020100 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2016020100 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2016020112 b/internal/tests/data/tc_gen/genesis/genesis_fake_2016020112 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2016020112 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2016020112 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2016020200 b/internal/tests/data/tc_gen/genesis/genesis_fake_2016020200 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2016020200 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2016020200 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2016020212 b/internal/tests/data/tc_gen/genesis/genesis_fake_2016020212 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2016020212 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2016020212 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2016103100 b/internal/tests/data/tc_gen/genesis/genesis_fake_2016103100 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2016103100 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2016103100 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2016103112 b/internal/tests/data/tc_gen/genesis/genesis_fake_2016103112 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2016103112 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2016103112 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2017020100 b/internal/tests/data/tc_gen/genesis/genesis_fake_2017020100 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2017020100 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2017020100 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2017020112 b/internal/tests/data/tc_gen/genesis/genesis_fake_2017020112 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2017020112 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2017020112 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2017020200 b/internal/tests/data/tc_gen/genesis/genesis_fake_2017020200 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2017020200 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2017020200 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2017020212 b/internal/tests/data/tc_gen/genesis/genesis_fake_2017020212 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2017020212 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2017020212 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2017103100 b/internal/tests/data/tc_gen/genesis/genesis_fake_2017103100 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2017103100 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2017103100 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2017103112 b/internal/tests/data/tc_gen/genesis/genesis_fake_2017103112 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2017103112 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2017103112 diff --git a/internal_tests/data/tc_gen/genesis/genesis_fake_2017123018 b/internal/tests/data/tc_gen/genesis/genesis_fake_2017123018 similarity index 100% rename from internal_tests/data/tc_gen/genesis/genesis_fake_2017123018 rename to internal/tests/data/tc_gen/genesis/genesis_fake_2017123018 diff --git a/internal_tests/data/tc_gen/shape/shape_fake_2016020100 b/internal/tests/data/tc_gen/shape/shape_fake_2016020100 similarity index 100% rename from internal_tests/data/tc_gen/shape/shape_fake_2016020100 rename to internal/tests/data/tc_gen/shape/shape_fake_2016020100 diff --git a/internal_tests/data/tc_gen/shape/shape_fake_2016020112 b/internal/tests/data/tc_gen/shape/shape_fake_2016020112 similarity index 100% rename from internal_tests/data/tc_gen/shape/shape_fake_2016020112 rename to internal/tests/data/tc_gen/shape/shape_fake_2016020112 diff --git a/internal_tests/data/tc_gen/shape/shape_fake_2016020200 b/internal/tests/data/tc_gen/shape/shape_fake_2016020200 similarity index 100% rename from internal_tests/data/tc_gen/shape/shape_fake_2016020200 rename to internal/tests/data/tc_gen/shape/shape_fake_2016020200 diff --git a/internal_tests/data/tc_gen/shape/shape_fake_2016020212 b/internal/tests/data/tc_gen/shape/shape_fake_2016020212 similarity index 100% rename from internal_tests/data/tc_gen/shape/shape_fake_2016020212 rename to internal/tests/data/tc_gen/shape/shape_fake_2016020212 diff --git a/internal_tests/data/tc_gen/shape/shape_fake_2017020100 b/internal/tests/data/tc_gen/shape/shape_fake_2017020100 similarity index 100% rename from internal_tests/data/tc_gen/shape/shape_fake_2017020100 rename to internal/tests/data/tc_gen/shape/shape_fake_2017020100 diff --git a/internal_tests/data/tc_gen/shape/shape_fake_2017020112 b/internal/tests/data/tc_gen/shape/shape_fake_2017020112 similarity index 100% rename from internal_tests/data/tc_gen/shape/shape_fake_2017020112 rename to internal/tests/data/tc_gen/shape/shape_fake_2017020112 diff --git a/internal_tests/data/tc_gen/shape/shape_fake_2017020200 b/internal/tests/data/tc_gen/shape/shape_fake_2017020200 similarity index 100% rename from internal_tests/data/tc_gen/shape/shape_fake_2017020200 rename to internal/tests/data/tc_gen/shape/shape_fake_2017020200 diff --git a/internal_tests/data/tc_gen/shape/shape_fake_2017020212 b/internal/tests/data/tc_gen/shape/shape_fake_2017020212 similarity index 100% rename from internal_tests/data/tc_gen/shape/shape_fake_2017020212 rename to internal/tests/data/tc_gen/shape/shape_fake_2017020212 diff --git a/internal_tests/data/tc_gen/shape/shape_fake_2017103100 b/internal/tests/data/tc_gen/shape/shape_fake_2017103100 similarity index 100% rename from internal_tests/data/tc_gen/shape/shape_fake_2017103100 rename to internal/tests/data/tc_gen/shape/shape_fake_2017103100 diff --git a/internal_tests/data/tc_gen/shape/shape_fake_2017103112 b/internal/tests/data/tc_gen/shape/shape_fake_2017103112 similarity index 100% rename from internal_tests/data/tc_gen/shape/shape_fake_2017103112 rename to internal/tests/data/tc_gen/shape/shape_fake_2017103112 diff --git a/internal_tests/data/tc_gen/shape/shape_fake_2017123018 b/internal/tests/data/tc_gen/shape/shape_fake_2017123018 similarity index 100% rename from internal_tests/data/tc_gen/shape/shape_fake_2017123018 rename to internal/tests/data/tc_gen/shape/shape_fake_2017123018 diff --git a/internal_tests/data/tc_gen/track/track_fake_2016020100 b/internal/tests/data/tc_gen/track/track_fake_2016020100 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2016020100 rename to internal/tests/data/tc_gen/track/track_fake_2016020100 diff --git a/internal_tests/data/tc_gen/track/track_fake_2016020112 b/internal/tests/data/tc_gen/track/track_fake_2016020112 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2016020112 rename to internal/tests/data/tc_gen/track/track_fake_2016020112 diff --git a/internal_tests/data/tc_gen/track/track_fake_2016020200 b/internal/tests/data/tc_gen/track/track_fake_2016020200 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2016020200 rename to internal/tests/data/tc_gen/track/track_fake_2016020200 diff --git a/internal_tests/data/tc_gen/track/track_fake_2016020212 b/internal/tests/data/tc_gen/track/track_fake_2016020212 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2016020212 rename to internal/tests/data/tc_gen/track/track_fake_2016020212 diff --git a/internal_tests/data/tc_gen/track/track_fake_2016103100 b/internal/tests/data/tc_gen/track/track_fake_2016103100 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2016103100 rename to internal/tests/data/tc_gen/track/track_fake_2016103100 diff --git a/internal_tests/data/tc_gen/track/track_fake_2016103112 b/internal/tests/data/tc_gen/track/track_fake_2016103112 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2016103112 rename to internal/tests/data/tc_gen/track/track_fake_2016103112 diff --git a/internal_tests/data/tc_gen/track/track_fake_2017020100 b/internal/tests/data/tc_gen/track/track_fake_2017020100 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2017020100 rename to internal/tests/data/tc_gen/track/track_fake_2017020100 diff --git a/internal_tests/data/tc_gen/track/track_fake_2017020112 b/internal/tests/data/tc_gen/track/track_fake_2017020112 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2017020112 rename to internal/tests/data/tc_gen/track/track_fake_2017020112 diff --git a/internal_tests/data/tc_gen/track/track_fake_2017020200 b/internal/tests/data/tc_gen/track/track_fake_2017020200 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2017020200 rename to internal/tests/data/tc_gen/track/track_fake_2017020200 diff --git a/internal_tests/data/tc_gen/track/track_fake_2017020212 b/internal/tests/data/tc_gen/track/track_fake_2017020212 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2017020212 rename to internal/tests/data/tc_gen/track/track_fake_2017020212 diff --git a/internal_tests/data/tc_gen/track/track_fake_2017103100 b/internal/tests/data/tc_gen/track/track_fake_2017103100 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2017103100 rename to internal/tests/data/tc_gen/track/track_fake_2017103100 diff --git a/internal_tests/data/tc_gen/track/track_fake_2017103112 b/internal/tests/data/tc_gen/track/track_fake_2017103112 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2017103112 rename to internal/tests/data/tc_gen/track/track_fake_2017103112 diff --git a/internal_tests/data/tc_gen/track/track_fake_2017123018 b/internal/tests/data/tc_gen/track/track_fake_2017123018 similarity index 100% rename from internal_tests/data/tc_gen/track/track_fake_2017123018 rename to internal/tests/data/tc_gen/track/track_fake_2017123018 diff --git a/internal_tests/data/tc_pairs/adeck/amlq2014123118.gfso.0104 b/internal/tests/data/tc_pairs/adeck/amlq2014123118.gfso.0104 similarity index 100% rename from internal_tests/data/tc_pairs/adeck/amlq2014123118.gfso.0104 rename to internal/tests/data/tc_pairs/adeck/amlq2014123118.gfso.0104 diff --git a/internal_tests/data/tc_pairs/adeck/fake_file b/internal/tests/data/tc_pairs/adeck/fake_file similarity index 100% rename from internal_tests/data/tc_pairs/adeck/fake_file rename to internal/tests/data/tc_pairs/adeck/fake_file diff --git a/internal_tests/data/tc_pairs/bdeck/bmlq2014123118.gfso.0104 b/internal/tests/data/tc_pairs/bdeck/bmlq2014123118.gfso.0104 similarity index 100% rename from internal_tests/data/tc_pairs/bdeck/bmlq2014123118.gfso.0104 rename to internal/tests/data/tc_pairs/bdeck/bmlq2014123118.gfso.0104 diff --git a/internal_tests/data/tc_pairs/bdeck/get_bdeck_balq2014123118.gfso.0104 b/internal/tests/data/tc_pairs/bdeck/get_bdeck_balq2014123118.gfso.0104 similarity index 100% rename from internal_tests/data/tc_pairs/bdeck/get_bdeck_balq2014123118.gfso.0104 rename to internal/tests/data/tc_pairs/bdeck/get_bdeck_balq2014123118.gfso.0104 diff --git a/internal_tests/data/tc_pairs/bdeck/get_bdeck_bmlq2014123118.gfso.0104 b/internal/tests/data/tc_pairs/bdeck/get_bdeck_bmlq2014123118.gfso.0104 similarity index 100% rename from internal_tests/data/tc_pairs/bdeck/get_bdeck_bmlq2014123118.gfso.0104 rename to internal/tests/data/tc_pairs/bdeck/get_bdeck_bmlq2014123118.gfso.0104 diff --git a/internal_tests/data/tc_pairs/bdeck/get_bdeck_bmlq2014123118.gfso.0105 b/internal/tests/data/tc_pairs/bdeck/get_bdeck_bmlq2014123118.gfso.0105 similarity index 100% rename from internal_tests/data/tc_pairs/bdeck/get_bdeck_bmlq2014123118.gfso.0105 rename to internal/tests/data/tc_pairs/bdeck/get_bdeck_bmlq2014123118.gfso.0105 diff --git a/internal_tests/data/tc_pairs/bdeck/get_bdeck_bmlq2019123118.gfso.0104 b/internal/tests/data/tc_pairs/bdeck/get_bdeck_bmlq2019123118.gfso.0104 similarity index 100% rename from internal_tests/data/tc_pairs/bdeck/get_bdeck_bmlq2019123118.gfso.0104 rename to internal/tests/data/tc_pairs/bdeck/get_bdeck_bmlq2019123118.gfso.0104 diff --git a/internal_tests/data/tc_pairs/bdeck/storm_id_bal092019.dat b/internal/tests/data/tc_pairs/bdeck/storm_id_bal092019.dat similarity index 100% rename from internal_tests/data/tc_pairs/bdeck/storm_id_bal092019.dat rename to internal/tests/data/tc_pairs/bdeck/storm_id_bal092019.dat diff --git a/internal_tests/data/tc_pairs/bdeck/storm_id_bal102019.dat b/internal/tests/data/tc_pairs/bdeck/storm_id_bal102019.dat similarity index 100% rename from internal_tests/data/tc_pairs/bdeck/storm_id_bal102019.dat rename to internal/tests/data/tc_pairs/bdeck/storm_id_bal102019.dat diff --git a/internal_tests/data/tc_pairs/bdeck/storm_id_bml092019.dat b/internal/tests/data/tc_pairs/bdeck/storm_id_bml092019.dat similarity index 100% rename from internal_tests/data/tc_pairs/bdeck/storm_id_bml092019.dat rename to internal/tests/data/tc_pairs/bdeck/storm_id_bml092019.dat diff --git a/internal_tests/data/tc_pairs/bdeck/storm_id_bml102019.dat b/internal/tests/data/tc_pairs/bdeck/storm_id_bml102019.dat similarity index 100% rename from internal_tests/data/tc_pairs/bdeck/storm_id_bml102019.dat rename to internal/tests/data/tc_pairs/bdeck/storm_id_bml102019.dat diff --git a/internal_tests/data/tc_pairs/edeck/emlq2014123118.gfso.0104 b/internal/tests/data/tc_pairs/edeck/emlq2014123118.gfso.0104 similarity index 100% rename from internal_tests/data/tc_pairs/edeck/emlq2014123118.gfso.0104 rename to internal/tests/data/tc_pairs/edeck/emlq2014123118.gfso.0104 diff --git a/internal_tests/data/tc_pairs/edeck/storm_id_eal092019.dat b/internal/tests/data/tc_pairs/edeck/storm_id_eal092019.dat similarity index 100% rename from internal_tests/data/tc_pairs/edeck/storm_id_eal092019.dat rename to internal/tests/data/tc_pairs/edeck/storm_id_eal092019.dat diff --git a/internal_tests/data/tc_pairs/edeck/storm_id_eal102019.dat b/internal/tests/data/tc_pairs/edeck/storm_id_eal102019.dat similarity index 100% rename from internal_tests/data/tc_pairs/edeck/storm_id_eal102019.dat rename to internal/tests/data/tc_pairs/edeck/storm_id_eal102019.dat diff --git a/internal_tests/data/tc_pairs/edeck/storm_id_eml092019.dat b/internal/tests/data/tc_pairs/edeck/storm_id_eml092019.dat similarity index 100% rename from internal_tests/data/tc_pairs/edeck/storm_id_eml092019.dat rename to internal/tests/data/tc_pairs/edeck/storm_id_eml092019.dat diff --git a/internal_tests/data/tc_pairs/edeck/storm_id_eml102019.dat b/internal/tests/data/tc_pairs/edeck/storm_id_eml102019.dat similarity index 100% rename from internal_tests/data/tc_pairs/edeck/storm_id_eml102019.dat rename to internal/tests/data/tc_pairs/edeck/storm_id_eml102019.dat diff --git a/internal_tests/data/tiles/fcst/20141214_00/ML1201072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc b/internal/tests/data/tiles/fcst/20141214_00/ML1201072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc similarity index 100% rename from internal_tests/data/tiles/fcst/20141214_00/ML1201072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc rename to internal/tests/data/tiles/fcst/20141214_00/ML1201072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc diff --git a/internal_tests/data/tiles/fcst/20141214_00/ML1201072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc b/internal/tests/data/tiles/fcst/20141214_00/ML1201072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc similarity index 100% rename from internal_tests/data/tiles/fcst/20141214_00/ML1201072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc rename to internal/tests/data/tiles/fcst/20141214_00/ML1201072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc diff --git a/internal_tests/data/tiles/fcst/20141214_00/ML1201072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc b/internal/tests/data/tiles/fcst/20141214_00/ML1201072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc similarity index 100% rename from internal_tests/data/tiles/fcst/20141214_00/ML1201072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc rename to internal/tests/data/tiles/fcst/20141214_00/ML1201072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc diff --git a/internal_tests/data/tiles/fcst/20141214_00/ML1221072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc b/internal/tests/data/tiles/fcst/20141214_00/ML1221072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc similarity index 100% rename from internal_tests/data/tiles/fcst/20141214_00/ML1221072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc rename to internal/tests/data/tiles/fcst/20141214_00/ML1221072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc diff --git a/internal_tests/data/tiles/fcst/20141214_00/ML1221072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc b/internal/tests/data/tiles/fcst/20141214_00/ML1221072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc similarity index 100% rename from internal_tests/data/tiles/fcst/20141214_00/ML1221072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc rename to internal/tests/data/tiles/fcst/20141214_00/ML1221072014/FCST_TILE_F006_gfs_4_20141214_0000_006.nc diff --git a/internal_tests/data/tiles/fcst/20141214_00/ML1221072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc b/internal/tests/data/tiles/fcst/20141214_00/ML1221072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc similarity index 100% rename from internal_tests/data/tiles/fcst/20141214_00/ML1221072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc rename to internal/tests/data/tiles/fcst/20141214_00/ML1221072014/FCST_TILE_F012_gfs_4_20141214_0000_012.nc diff --git a/internal_tests/data/tiles/fcst/20141215_00/ML1291072014/FCST_TILE_F000_gfs_4_20141215_0000_000.nc b/internal/tests/data/tiles/fcst/20141215_00/ML1291072014/FCST_TILE_F000_gfs_4_20141215_0000_000.nc similarity index 100% rename from internal_tests/data/tiles/fcst/20141215_00/ML1291072014/FCST_TILE_F000_gfs_4_20141215_0000_000.nc rename to internal/tests/data/tiles/fcst/20141215_00/ML1291072014/FCST_TILE_F000_gfs_4_20141215_0000_000.nc diff --git a/internal_tests/data/tiles/fcst/20141215_00/ML1291072014/FCST_TILE_F006_gfs_4_20141215_0000_006.nc b/internal/tests/data/tiles/fcst/20141215_00/ML1291072014/FCST_TILE_F006_gfs_4_20141215_0000_006.nc similarity index 100% rename from internal_tests/data/tiles/fcst/20141215_00/ML1291072014/FCST_TILE_F006_gfs_4_20141215_0000_006.nc rename to internal/tests/data/tiles/fcst/20141215_00/ML1291072014/FCST_TILE_F006_gfs_4_20141215_0000_006.nc diff --git a/internal_tests/data/tiles/fcst/20141215_00/ML1291072014/FCST_TILE_F012_gfs_4_20141215_0000_012.nc b/internal/tests/data/tiles/fcst/20141215_00/ML1291072014/FCST_TILE_F012_gfs_4_20141215_0000_012.nc similarity index 100% rename from internal_tests/data/tiles/fcst/20141215_00/ML1291072014/FCST_TILE_F012_gfs_4_20141215_0000_012.nc rename to internal/tests/data/tiles/fcst/20141215_00/ML1291072014/FCST_TILE_F012_gfs_4_20141215_0000_012.nc diff --git a/internal_tests/data/tiles/obs/20141214_00/ML1201072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc b/internal/tests/data/tiles/obs/20141214_00/ML1201072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc similarity index 100% rename from internal_tests/data/tiles/obs/20141214_00/ML1201072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc rename to internal/tests/data/tiles/obs/20141214_00/ML1201072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc diff --git a/internal_tests/data/tiles/obs/20141214_00/ML1201072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc b/internal/tests/data/tiles/obs/20141214_00/ML1201072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc similarity index 100% rename from internal_tests/data/tiles/obs/20141214_00/ML1201072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc rename to internal/tests/data/tiles/obs/20141214_00/ML1201072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc diff --git a/internal_tests/data/tiles/obs/20141214_00/ML1201072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc b/internal/tests/data/tiles/obs/20141214_00/ML1201072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc similarity index 100% rename from internal_tests/data/tiles/obs/20141214_00/ML1201072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc rename to internal/tests/data/tiles/obs/20141214_00/ML1201072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc diff --git a/internal_tests/data/tiles/obs/20141214_00/ML1221072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc b/internal/tests/data/tiles/obs/20141214_00/ML1221072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc similarity index 100% rename from internal_tests/data/tiles/obs/20141214_00/ML1221072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc rename to internal/tests/data/tiles/obs/20141214_00/ML1221072014/OBS_TILE_F000_gfs_4_20141214_0000_000.nc diff --git a/internal_tests/data/tiles/obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc b/internal/tests/data/tiles/obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc similarity index 100% rename from internal_tests/data/tiles/obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc rename to internal/tests/data/tiles/obs/20141214_00/ML1221072014/OBS_TILE_F006_gfs_4_20141214_0000_006.nc diff --git a/internal_tests/data/tiles/obs/20141214_00/ML1221072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc b/internal/tests/data/tiles/obs/20141214_00/ML1221072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc similarity index 100% rename from internal_tests/data/tiles/obs/20141214_00/ML1221072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc rename to internal/tests/data/tiles/obs/20141214_00/ML1221072014/OBS_TILE_F012_gfs_4_20141214_0000_012.nc diff --git a/internal_tests/data/tiles/obs/20141215_00/ML1291072014/OBS_TILE_F000_gfs_4_20141215_0000_000.nc b/internal/tests/data/tiles/obs/20141215_00/ML1291072014/OBS_TILE_F000_gfs_4_20141215_0000_000.nc similarity index 100% rename from internal_tests/data/tiles/obs/20141215_00/ML1291072014/OBS_TILE_F000_gfs_4_20141215_0000_000.nc rename to internal/tests/data/tiles/obs/20141215_00/ML1291072014/OBS_TILE_F000_gfs_4_20141215_0000_000.nc diff --git a/internal_tests/data/tiles/obs/20141215_00/ML1291072014/OBS_TILE_F006_gfs_4_20141215_0000_006.nc b/internal/tests/data/tiles/obs/20141215_00/ML1291072014/OBS_TILE_F006_gfs_4_20141215_0000_006.nc similarity index 100% rename from internal_tests/data/tiles/obs/20141215_00/ML1291072014/OBS_TILE_F006_gfs_4_20141215_0000_006.nc rename to internal/tests/data/tiles/obs/20141215_00/ML1291072014/OBS_TILE_F006_gfs_4_20141215_0000_006.nc diff --git a/internal_tests/data/tiles/obs/20141215_00/ML1291072014/OBS_TILE_F012_gfs_4_20141215_0000_012.nc b/internal/tests/data/tiles/obs/20141215_00/ML1291072014/OBS_TILE_F012_gfs_4_20141215_0000_012.nc similarity index 100% rename from internal_tests/data/tiles/obs/20141215_00/ML1291072014/OBS_TILE_F012_gfs_4_20141215_0000_012.nc rename to internal/tests/data/tiles/obs/20141215_00/ML1291072014/OBS_TILE_F012_gfs_4_20141215_0000_012.nc diff --git a/internal_tests/data/user_script/init_20141024093015_valid_20141031093015_lead_168.nc b/internal/tests/data/user_script/init_20141024093015_valid_20141031093015_lead_168.nc similarity index 100% rename from internal_tests/data/user_script/init_20141024093015_valid_20141031093015_lead_168.nc rename to internal/tests/data/user_script/init_20141024093015_valid_20141031093015_lead_168.nc diff --git a/internal_tests/data/user_script/init_20141024213015_valid_20141031213015_lead_168.nc b/internal/tests/data/user_script/init_20141024213015_valid_20141031213015_lead_168.nc similarity index 100% rename from internal_tests/data/user_script/init_20141024213015_valid_20141031213015_lead_168.nc rename to internal/tests/data/user_script/init_20141024213015_valid_20141031213015_lead_168.nc diff --git a/internal_tests/data/user_script/init_20141025093015_valid_20141031093015_lead_144.nc b/internal/tests/data/user_script/init_20141025093015_valid_20141031093015_lead_144.nc similarity index 100% rename from internal_tests/data/user_script/init_20141025093015_valid_20141031093015_lead_144.nc rename to internal/tests/data/user_script/init_20141025093015_valid_20141031093015_lead_144.nc diff --git a/internal_tests/data/user_script/init_20141025093015_valid_20141101093015_lead_168.nc b/internal/tests/data/user_script/init_20141025093015_valid_20141101093015_lead_168.nc similarity index 100% rename from internal_tests/data/user_script/init_20141025093015_valid_20141101093015_lead_168.nc rename to internal/tests/data/user_script/init_20141025093015_valid_20141101093015_lead_168.nc diff --git a/internal_tests/data/user_script/init_20141025213015_valid_20141031213015_lead_144.nc b/internal/tests/data/user_script/init_20141025213015_valid_20141031213015_lead_144.nc similarity index 100% rename from internal_tests/data/user_script/init_20141025213015_valid_20141031213015_lead_144.nc rename to internal/tests/data/user_script/init_20141025213015_valid_20141031213015_lead_144.nc diff --git a/internal_tests/data/user_script/init_20141026093015_valid_20141031093015_lead_120.nc b/internal/tests/data/user_script/init_20141026093015_valid_20141031093015_lead_120.nc similarity index 100% rename from internal_tests/data/user_script/init_20141026093015_valid_20141031093015_lead_120.nc rename to internal/tests/data/user_script/init_20141026093015_valid_20141031093015_lead_120.nc diff --git a/internal_tests/data/user_script/init_20141026093015_valid_20141101093015_lead_144.nc b/internal/tests/data/user_script/init_20141026093015_valid_20141101093015_lead_144.nc similarity index 100% rename from internal_tests/data/user_script/init_20141026093015_valid_20141101093015_lead_144.nc rename to internal/tests/data/user_script/init_20141026093015_valid_20141101093015_lead_144.nc diff --git a/internal_tests/data/user_script/init_20141026213015_valid_20141031213015_lead_120.nc b/internal/tests/data/user_script/init_20141026213015_valid_20141031213015_lead_120.nc similarity index 100% rename from internal_tests/data/user_script/init_20141026213015_valid_20141031213015_lead_120.nc rename to internal/tests/data/user_script/init_20141026213015_valid_20141031213015_lead_120.nc diff --git a/internal_tests/data/user_script/init_20141027093015_valid_20141031093015_lead_096.nc b/internal/tests/data/user_script/init_20141027093015_valid_20141031093015_lead_096.nc similarity index 100% rename from internal_tests/data/user_script/init_20141027093015_valid_20141031093015_lead_096.nc rename to internal/tests/data/user_script/init_20141027093015_valid_20141031093015_lead_096.nc diff --git a/internal_tests/data/user_script/init_20141027093015_valid_20141101093015_lead_120.nc b/internal/tests/data/user_script/init_20141027093015_valid_20141101093015_lead_120.nc similarity index 100% rename from internal_tests/data/user_script/init_20141027093015_valid_20141101093015_lead_120.nc rename to internal/tests/data/user_script/init_20141027093015_valid_20141101093015_lead_120.nc diff --git a/internal_tests/data/user_script/init_20141027213015_valid_20141031213015_lead_096.nc b/internal/tests/data/user_script/init_20141027213015_valid_20141031213015_lead_096.nc similarity index 100% rename from internal_tests/data/user_script/init_20141027213015_valid_20141031213015_lead_096.nc rename to internal/tests/data/user_script/init_20141027213015_valid_20141031213015_lead_096.nc diff --git a/internal_tests/data/user_script/init_20141028093015_valid_20141031093015_lead_072.nc b/internal/tests/data/user_script/init_20141028093015_valid_20141031093015_lead_072.nc similarity index 100% rename from internal_tests/data/user_script/init_20141028093015_valid_20141031093015_lead_072.nc rename to internal/tests/data/user_script/init_20141028093015_valid_20141031093015_lead_072.nc diff --git a/internal_tests/data/user_script/init_20141028093015_valid_20141101093015_lead_096.nc b/internal/tests/data/user_script/init_20141028093015_valid_20141101093015_lead_096.nc similarity index 100% rename from internal_tests/data/user_script/init_20141028093015_valid_20141101093015_lead_096.nc rename to internal/tests/data/user_script/init_20141028093015_valid_20141101093015_lead_096.nc diff --git a/internal_tests/data/user_script/init_20141028213015_valid_20141031213015_lead_072.nc b/internal/tests/data/user_script/init_20141028213015_valid_20141031213015_lead_072.nc similarity index 100% rename from internal_tests/data/user_script/init_20141028213015_valid_20141031213015_lead_072.nc rename to internal/tests/data/user_script/init_20141028213015_valid_20141031213015_lead_072.nc diff --git a/internal_tests/data/user_script/init_20141029093015_valid_20141031093015_lead_048.nc b/internal/tests/data/user_script/init_20141029093015_valid_20141031093015_lead_048.nc similarity index 100% rename from internal_tests/data/user_script/init_20141029093015_valid_20141031093015_lead_048.nc rename to internal/tests/data/user_script/init_20141029093015_valid_20141031093015_lead_048.nc diff --git a/internal_tests/data/user_script/init_20141029093015_valid_20141101093015_lead_072.nc b/internal/tests/data/user_script/init_20141029093015_valid_20141101093015_lead_072.nc similarity index 100% rename from internal_tests/data/user_script/init_20141029093015_valid_20141101093015_lead_072.nc rename to internal/tests/data/user_script/init_20141029093015_valid_20141101093015_lead_072.nc diff --git a/internal_tests/data/user_script/init_20141029213015_valid_20141031213015_lead_048.nc b/internal/tests/data/user_script/init_20141029213015_valid_20141031213015_lead_048.nc similarity index 100% rename from internal_tests/data/user_script/init_20141029213015_valid_20141031213015_lead_048.nc rename to internal/tests/data/user_script/init_20141029213015_valid_20141031213015_lead_048.nc diff --git a/internal_tests/data/user_script/init_20141030093015_valid_20141031093015_lead_024.nc b/internal/tests/data/user_script/init_20141030093015_valid_20141031093015_lead_024.nc similarity index 100% rename from internal_tests/data/user_script/init_20141030093015_valid_20141031093015_lead_024.nc rename to internal/tests/data/user_script/init_20141030093015_valid_20141031093015_lead_024.nc diff --git a/internal_tests/data/user_script/init_20141030093015_valid_20141101093015_lead_048.nc b/internal/tests/data/user_script/init_20141030093015_valid_20141101093015_lead_048.nc similarity index 100% rename from internal_tests/data/user_script/init_20141030093015_valid_20141101093015_lead_048.nc rename to internal/tests/data/user_script/init_20141030093015_valid_20141101093015_lead_048.nc diff --git a/internal_tests/data/user_script/init_20141030213015_valid_20141031093015_lead_012.nc b/internal/tests/data/user_script/init_20141030213015_valid_20141031093015_lead_012.nc similarity index 100% rename from internal_tests/data/user_script/init_20141030213015_valid_20141031093015_lead_012.nc rename to internal/tests/data/user_script/init_20141030213015_valid_20141031093015_lead_012.nc diff --git a/internal_tests/data/user_script/init_20141030213015_valid_20141031213015_lead_024.nc b/internal/tests/data/user_script/init_20141030213015_valid_20141031213015_lead_024.nc similarity index 100% rename from internal_tests/data/user_script/init_20141030213015_valid_20141031213015_lead_024.nc rename to internal/tests/data/user_script/init_20141030213015_valid_20141031213015_lead_024.nc diff --git a/internal_tests/data/user_script/init_20141031093015_valid_20141031093015_lead_000.nc b/internal/tests/data/user_script/init_20141031093015_valid_20141031093015_lead_000.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031093015_valid_20141031093015_lead_000.nc rename to internal/tests/data/user_script/init_20141031093015_valid_20141031093015_lead_000.nc diff --git a/internal_tests/data/user_script/init_20141031093015_valid_20141031213015_lead_012.nc b/internal/tests/data/user_script/init_20141031093015_valid_20141031213015_lead_012.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031093015_valid_20141031213015_lead_012.nc rename to internal/tests/data/user_script/init_20141031093015_valid_20141031213015_lead_012.nc diff --git a/internal_tests/data/user_script/init_20141031093015_valid_20141101093015_lead_024.nc b/internal/tests/data/user_script/init_20141031093015_valid_20141101093015_lead_024.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031093015_valid_20141101093015_lead_024.nc rename to internal/tests/data/user_script/init_20141031093015_valid_20141101093015_lead_024.nc diff --git a/internal_tests/data/user_script/init_20141031093015_valid_20141102093015_lead_048.nc b/internal/tests/data/user_script/init_20141031093015_valid_20141102093015_lead_048.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031093015_valid_20141102093015_lead_048.nc rename to internal/tests/data/user_script/init_20141031093015_valid_20141102093015_lead_048.nc diff --git a/internal_tests/data/user_script/init_20141031093015_valid_20141103093015_lead_072.nc b/internal/tests/data/user_script/init_20141031093015_valid_20141103093015_lead_072.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031093015_valid_20141103093015_lead_072.nc rename to internal/tests/data/user_script/init_20141031093015_valid_20141103093015_lead_072.nc diff --git a/internal_tests/data/user_script/init_20141031093015_valid_20141104093015_lead_096.nc b/internal/tests/data/user_script/init_20141031093015_valid_20141104093015_lead_096.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031093015_valid_20141104093015_lead_096.nc rename to internal/tests/data/user_script/init_20141031093015_valid_20141104093015_lead_096.nc diff --git a/internal_tests/data/user_script/init_20141031093015_valid_20141105093015_lead_120.nc b/internal/tests/data/user_script/init_20141031093015_valid_20141105093015_lead_120.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031093015_valid_20141105093015_lead_120.nc rename to internal/tests/data/user_script/init_20141031093015_valid_20141105093015_lead_120.nc diff --git a/internal_tests/data/user_script/init_20141031093015_valid_20141106093015_lead_144.nc b/internal/tests/data/user_script/init_20141031093015_valid_20141106093015_lead_144.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031093015_valid_20141106093015_lead_144.nc rename to internal/tests/data/user_script/init_20141031093015_valid_20141106093015_lead_144.nc diff --git a/internal_tests/data/user_script/init_20141031093015_valid_20141107093015_lead_168.nc b/internal/tests/data/user_script/init_20141031093015_valid_20141107093015_lead_168.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031093015_valid_20141107093015_lead_168.nc rename to internal/tests/data/user_script/init_20141031093015_valid_20141107093015_lead_168.nc diff --git a/internal_tests/data/user_script/init_20141031213015_valid_20141031213015_lead_000.nc b/internal/tests/data/user_script/init_20141031213015_valid_20141031213015_lead_000.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031213015_valid_20141031213015_lead_000.nc rename to internal/tests/data/user_script/init_20141031213015_valid_20141031213015_lead_000.nc diff --git a/internal_tests/data/user_script/init_20141031213015_valid_20141101093015_lead_012.nc b/internal/tests/data/user_script/init_20141031213015_valid_20141101093015_lead_012.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031213015_valid_20141101093015_lead_012.nc rename to internal/tests/data/user_script/init_20141031213015_valid_20141101093015_lead_012.nc diff --git a/internal_tests/data/user_script/init_20141031213015_valid_20141101213015_lead_024.nc b/internal/tests/data/user_script/init_20141031213015_valid_20141101213015_lead_024.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031213015_valid_20141101213015_lead_024.nc rename to internal/tests/data/user_script/init_20141031213015_valid_20141101213015_lead_024.nc diff --git a/internal_tests/data/user_script/init_20141031213015_valid_20141102213015_lead_048.nc b/internal/tests/data/user_script/init_20141031213015_valid_20141102213015_lead_048.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031213015_valid_20141102213015_lead_048.nc rename to internal/tests/data/user_script/init_20141031213015_valid_20141102213015_lead_048.nc diff --git a/internal_tests/data/user_script/init_20141031213015_valid_20141103213015_lead_072.nc b/internal/tests/data/user_script/init_20141031213015_valid_20141103213015_lead_072.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031213015_valid_20141103213015_lead_072.nc rename to internal/tests/data/user_script/init_20141031213015_valid_20141103213015_lead_072.nc diff --git a/internal_tests/data/user_script/init_20141031213015_valid_20141104213015_lead_096.nc b/internal/tests/data/user_script/init_20141031213015_valid_20141104213015_lead_096.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031213015_valid_20141104213015_lead_096.nc rename to internal/tests/data/user_script/init_20141031213015_valid_20141104213015_lead_096.nc diff --git a/internal_tests/data/user_script/init_20141031213015_valid_20141105213015_lead_120.nc b/internal/tests/data/user_script/init_20141031213015_valid_20141105213015_lead_120.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031213015_valid_20141105213015_lead_120.nc rename to internal/tests/data/user_script/init_20141031213015_valid_20141105213015_lead_120.nc diff --git a/internal_tests/data/user_script/init_20141031213015_valid_20141106213015_lead_144.nc b/internal/tests/data/user_script/init_20141031213015_valid_20141106213015_lead_144.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031213015_valid_20141106213015_lead_144.nc rename to internal/tests/data/user_script/init_20141031213015_valid_20141106213015_lead_144.nc diff --git a/internal_tests/data/user_script/init_20141031213015_valid_20141107213015_lead_168.nc b/internal/tests/data/user_script/init_20141031213015_valid_20141107213015_lead_168.nc similarity index 100% rename from internal_tests/data/user_script/init_20141031213015_valid_20141107213015_lead_168.nc rename to internal/tests/data/user_script/init_20141031213015_valid_20141107213015_lead_168.nc diff --git a/internal_tests/data/user_script/init_20141101093015_valid_20141101093015_lead_000.nc b/internal/tests/data/user_script/init_20141101093015_valid_20141101093015_lead_000.nc similarity index 100% rename from internal_tests/data/user_script/init_20141101093015_valid_20141101093015_lead_000.nc rename to internal/tests/data/user_script/init_20141101093015_valid_20141101093015_lead_000.nc diff --git a/internal_tests/data/user_script/init_20141101093015_valid_20141101213015_lead_012.nc b/internal/tests/data/user_script/init_20141101093015_valid_20141101213015_lead_012.nc similarity index 100% rename from internal_tests/data/user_script/init_20141101093015_valid_20141101213015_lead_012.nc rename to internal/tests/data/user_script/init_20141101093015_valid_20141101213015_lead_012.nc diff --git a/internal_tests/data/user_script/init_20141101093015_valid_20141102093015_lead_024.nc b/internal/tests/data/user_script/init_20141101093015_valid_20141102093015_lead_024.nc similarity index 100% rename from internal_tests/data/user_script/init_20141101093015_valid_20141102093015_lead_024.nc rename to internal/tests/data/user_script/init_20141101093015_valid_20141102093015_lead_024.nc diff --git a/internal_tests/data/user_script/init_20141101093015_valid_20141103093015_lead_048.nc b/internal/tests/data/user_script/init_20141101093015_valid_20141103093015_lead_048.nc similarity index 100% rename from internal_tests/data/user_script/init_20141101093015_valid_20141103093015_lead_048.nc rename to internal/tests/data/user_script/init_20141101093015_valid_20141103093015_lead_048.nc diff --git a/internal_tests/data/user_script/init_20141101093015_valid_20141104093015_lead_072.nc b/internal/tests/data/user_script/init_20141101093015_valid_20141104093015_lead_072.nc similarity index 100% rename from internal_tests/data/user_script/init_20141101093015_valid_20141104093015_lead_072.nc rename to internal/tests/data/user_script/init_20141101093015_valid_20141104093015_lead_072.nc diff --git a/internal_tests/data/user_script/init_20141101093015_valid_20141105093015_lead_096.nc b/internal/tests/data/user_script/init_20141101093015_valid_20141105093015_lead_096.nc similarity index 100% rename from internal_tests/data/user_script/init_20141101093015_valid_20141105093015_lead_096.nc rename to internal/tests/data/user_script/init_20141101093015_valid_20141105093015_lead_096.nc diff --git a/internal_tests/data/user_script/init_20141101093015_valid_20141106093015_lead_120.nc b/internal/tests/data/user_script/init_20141101093015_valid_20141106093015_lead_120.nc similarity index 100% rename from internal_tests/data/user_script/init_20141101093015_valid_20141106093015_lead_120.nc rename to internal/tests/data/user_script/init_20141101093015_valid_20141106093015_lead_120.nc diff --git a/internal_tests/data/user_script/init_20141101093015_valid_20141107093015_lead_144.nc b/internal/tests/data/user_script/init_20141101093015_valid_20141107093015_lead_144.nc similarity index 100% rename from internal_tests/data/user_script/init_20141101093015_valid_20141107093015_lead_144.nc rename to internal/tests/data/user_script/init_20141101093015_valid_20141107093015_lead_144.nc diff --git a/internal_tests/data/user_script/init_20141101093015_valid_20141108093015_lead_168.nc b/internal/tests/data/user_script/init_20141101093015_valid_20141108093015_lead_168.nc similarity index 100% rename from internal_tests/data/user_script/init_20141101093015_valid_20141108093015_lead_168.nc rename to internal/tests/data/user_script/init_20141101093015_valid_20141108093015_lead_168.nc diff --git a/internal_tests/data/zip/testfile.txt.gz b/internal/tests/data/zip/testfile.txt.gz similarity index 100% rename from internal_tests/data/zip/testfile.txt.gz rename to internal/tests/data/zip/testfile.txt.gz diff --git a/internal_tests/data/zip/testfile2.txt.bz2 b/internal/tests/data/zip/testfile2.txt.bz2 similarity index 100% rename from internal_tests/data/zip/testfile2.txt.bz2 rename to internal/tests/data/zip/testfile2.txt.bz2 diff --git a/internal_tests/data/zip/testfile3.txt.zip b/internal/tests/data/zip/testfile3.txt.zip similarity index 100% rename from internal_tests/data/zip/testfile3.txt.zip rename to internal/tests/data/zip/testfile3.txt.zip diff --git a/internal_tests/data/zip/testfile4.txt b/internal/tests/data/zip/testfile4.txt similarity index 100% rename from internal_tests/data/zip/testfile4.txt rename to internal/tests/data/zip/testfile4.txt diff --git a/internal_tests/plotting/examples/plot_emc_grid2grid_anom.conf b/internal/tests/plotting/examples/plot_emc_grid2grid_anom.conf similarity index 100% rename from internal_tests/plotting/examples/plot_emc_grid2grid_anom.conf rename to internal/tests/plotting/examples/plot_emc_grid2grid_anom.conf diff --git a/internal_tests/plotting/examples/plot_emc_grid2grid_pres.conf b/internal/tests/plotting/examples/plot_emc_grid2grid_pres.conf similarity index 100% rename from internal_tests/plotting/examples/plot_emc_grid2grid_pres.conf rename to internal/tests/plotting/examples/plot_emc_grid2grid_pres.conf diff --git a/internal_tests/plotting/examples/plot_emc_grid2grid_sfc.conf b/internal/tests/plotting/examples/plot_emc_grid2grid_sfc.conf similarity index 100% rename from internal_tests/plotting/examples/plot_emc_grid2grid_sfc.conf rename to internal/tests/plotting/examples/plot_emc_grid2grid_sfc.conf diff --git a/internal_tests/plotting/examples/plot_emc_grid2obs_conus_sfc.conf b/internal/tests/plotting/examples/plot_emc_grid2obs_conus_sfc.conf similarity index 100% rename from internal_tests/plotting/examples/plot_emc_grid2obs_conus_sfc.conf rename to internal/tests/plotting/examples/plot_emc_grid2obs_conus_sfc.conf diff --git a/internal_tests/plotting/examples/plot_emc_grid2obs_upper_air.conf b/internal/tests/plotting/examples/plot_emc_grid2obs_upper_air.conf similarity index 100% rename from internal_tests/plotting/examples/plot_emc_grid2obs_upper_air.conf rename to internal/tests/plotting/examples/plot_emc_grid2obs_upper_air.conf diff --git a/internal_tests/plotting/examples/plot_emc_precip_ccpa.conf b/internal/tests/plotting/examples/plot_emc_precip_ccpa.conf similarity index 100% rename from internal_tests/plotting/examples/plot_emc_precip_ccpa.conf rename to internal/tests/plotting/examples/plot_emc_precip_ccpa.conf diff --git a/internal_tests/plotting/examples/plot_user_plotting_scripts.conf b/internal/tests/plotting/examples/plot_user_plotting_scripts.conf similarity index 100% rename from internal_tests/plotting/examples/plot_user_plotting_scripts.conf rename to internal/tests/plotting/examples/plot_user_plotting_scripts.conf diff --git a/internal_tests/plotting/met_config/STATAnalysisConfig b/internal/tests/plotting/met_config/STATAnalysisConfig similarity index 100% rename from internal_tests/plotting/met_config/STATAnalysisConfig rename to internal/tests/plotting/met_config/STATAnalysisConfig diff --git a/internal_tests/pytests/conftest.py b/internal/tests/pytests/conftest.py similarity index 97% rename from internal_tests/pytests/conftest.py rename to internal/tests/pytests/conftest.py index eceb71941f..97af766988 100644 --- a/internal_tests/pytests/conftest.py +++ b/internal/tests/pytests/conftest.py @@ -7,7 +7,7 @@ from pathlib import Path # add METplus directory to path so the wrappers and utilities can be found -metplus_dir = str(Path(__file__).parents[2]) +metplus_dir = str(Path(__file__).parents[3]) sys.path.insert(0, os.path.abspath(metplus_dir)) from metplus.util import config_metplus diff --git a/internal_tests/pytests/minimum_pytest.conf b/internal/tests/pytests/minimum_pytest.conf similarity index 100% rename from internal_tests/pytests/minimum_pytest.conf rename to internal/tests/pytests/minimum_pytest.conf diff --git a/internal_tests/pytests/minimum_pytest.corrinado.sh b/internal/tests/pytests/minimum_pytest.corrinado.sh similarity index 100% rename from internal_tests/pytests/minimum_pytest.corrinado.sh rename to internal/tests/pytests/minimum_pytest.corrinado.sh diff --git a/internal_tests/pytests/minimum_pytest.dakota.sh b/internal/tests/pytests/minimum_pytest.dakota.sh similarity index 100% rename from internal_tests/pytests/minimum_pytest.dakota.sh rename to internal/tests/pytests/minimum_pytest.dakota.sh diff --git a/internal_tests/pytests/minimum_pytest.docker.sh b/internal/tests/pytests/minimum_pytest.docker.sh similarity index 100% rename from internal_tests/pytests/minimum_pytest.docker.sh rename to internal/tests/pytests/minimum_pytest.docker.sh diff --git a/internal_tests/pytests/minimum_pytest.eyewall.sh b/internal/tests/pytests/minimum_pytest.eyewall.sh similarity index 100% rename from internal_tests/pytests/minimum_pytest.eyewall.sh rename to internal/tests/pytests/minimum_pytest.eyewall.sh diff --git a/internal_tests/pytests/minimum_pytest.hera.sh b/internal/tests/pytests/minimum_pytest.hera.sh similarity index 100% rename from internal_tests/pytests/minimum_pytest.hera.sh rename to internal/tests/pytests/minimum_pytest.hera.sh diff --git a/internal_tests/pytests/minimum_pytest.kiowa.sh b/internal/tests/pytests/minimum_pytest.kiowa.sh similarity index 100% rename from internal_tests/pytests/minimum_pytest.kiowa.sh rename to internal/tests/pytests/minimum_pytest.kiowa.sh diff --git a/internal_tests/pytests/minimum_pytest.seneca.sh b/internal/tests/pytests/minimum_pytest.seneca.sh similarity index 100% rename from internal_tests/pytests/minimum_pytest.seneca.sh rename to internal/tests/pytests/minimum_pytest.seneca.sh diff --git a/internal_tests/pytests/minimum_pytest.venus.sh b/internal/tests/pytests/minimum_pytest.venus.sh similarity index 100% rename from internal_tests/pytests/minimum_pytest.venus.sh rename to internal/tests/pytests/minimum_pytest.venus.sh diff --git a/internal_tests/pytests/plotting/make_plots/test_make_plots.conf b/internal/tests/pytests/plotting/make_plots/test_make_plots.conf similarity index 100% rename from internal_tests/pytests/plotting/make_plots/test_make_plots.conf rename to internal/tests/pytests/plotting/make_plots/test_make_plots.conf diff --git a/internal_tests/pytests/plotting/make_plots/test_make_plots_wrapper.py b/internal/tests/pytests/plotting/make_plots/test_make_plots_wrapper.py similarity index 98% rename from internal_tests/pytests/plotting/make_plots/test_make_plots_wrapper.py rename to internal/tests/pytests/plotting/make_plots/test_make_plots_wrapper.py index 2be97153a6..5ee62c781a 100644 --- a/internal_tests/pytests/plotting/make_plots/test_make_plots_wrapper.py +++ b/internal/tests/pytests/plotting/make_plots/test_make_plots_wrapper.py @@ -6,7 +6,7 @@ from metplus.wrappers.make_plots_wrapper import MakePlotsWrapper -METPLUS_BASE = os.getcwd().split('/internal_tests')[0] +METPLUS_BASE = os.getcwd().split('/internal')[0] def make_plots_wrapper(metplus_config): diff --git a/internal_tests/pytests/plotting/plot_util/test_plot_util.py b/internal/tests/pytests/plotting/plot_util/test_plot_util.py similarity index 99% rename from internal_tests/pytests/plotting/plot_util/test_plot_util.py rename to internal/tests/pytests/plotting/plot_util/test_plot_util.py index d22ce6b7aa..8fb49fad8c 100644 --- a/internal_tests/pytests/plotting/plot_util/test_plot_util.py +++ b/internal/tests/pytests/plotting/plot_util/test_plot_util.py @@ -11,7 +11,7 @@ import pandas as pd -METPLUS_BASE = os.getcwd().split('/internal_tests')[0] +METPLUS_BASE = os.getcwd().split('/internal')[0] sys.path.append(METPLUS_BASE+'/ush/plotting_scripts') import plot_util logger = logging.getLogger('~/metplus_pytest_plot_util.log') diff --git a/internal_tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py b/internal/tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py similarity index 99% rename from internal_tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py rename to internal/tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py index 519bcbb943..23d3a27153 100644 --- a/internal_tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py +++ b/internal/tests/pytests/plotting/tcmpr_plotter/test_tcmpr_plotter.py @@ -190,7 +190,7 @@ def test_tcmpr_plotter_loop(metplus_config, config_overrides, config.set('config', key, value) test_data = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', + 'internal', 'tests', 'data', 'stat_data') @@ -287,7 +287,7 @@ def test_tcmpr_plotter(metplus_config, config_overrides, expected_string): config.set('config', key, value) test_data = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', + 'internal', 'tests', 'data', 'stat_data') if single_file: diff --git a/internal_tests/pytests/pytest.ini b/internal/tests/pytests/pytest.ini similarity index 100% rename from internal_tests/pytests/pytest.ini rename to internal/tests/pytests/pytest.ini diff --git a/internal_tests/pytests/util/config/config_1.conf b/internal/tests/pytests/util/config/config_1.conf similarity index 100% rename from internal_tests/pytests/util/config/config_1.conf rename to internal/tests/pytests/util/config/config_1.conf diff --git a/internal_tests/pytests/util/config/config_2.conf b/internal/tests/pytests/util/config/config_2.conf similarity index 100% rename from internal_tests/pytests/util/config/config_2.conf rename to internal/tests/pytests/util/config/config_2.conf diff --git a/internal_tests/pytests/util/config/config_3.conf b/internal/tests/pytests/util/config/config_3.conf similarity index 100% rename from internal_tests/pytests/util/config/config_3.conf rename to internal/tests/pytests/util/config/config_3.conf diff --git a/internal_tests/pytests/util/config/test_config.py b/internal/tests/pytests/util/config/test_config.py similarity index 100% rename from internal_tests/pytests/util/config/test_config.py rename to internal/tests/pytests/util/config/test_config.py diff --git a/internal_tests/pytests/util/config_metplus/test_config_metplus.py b/internal/tests/pytests/util/config_metplus/test_config_metplus.py similarity index 100% rename from internal_tests/pytests/util/config_metplus/test_config_metplus.py rename to internal/tests/pytests/util/config_metplus/test_config_metplus.py diff --git a/internal_tests/pytests/util/logging/test_logging.py b/internal/tests/pytests/util/logging/test_logging.py similarity index 100% rename from internal_tests/pytests/util/logging/test_logging.py rename to internal/tests/pytests/util/logging/test_logging.py diff --git a/internal_tests/pytests/util/met_config/test_met_config.py b/internal/tests/pytests/util/met_config/test_met_config.py similarity index 100% rename from internal_tests/pytests/util/met_config/test_met_config.py rename to internal/tests/pytests/util/met_config/test_met_config.py diff --git a/internal_tests/pytests/util/met_util/test_met_util.py b/internal/tests/pytests/util/met_util/test_met_util.py similarity index 98% rename from internal_tests/pytests/util/met_util/test_met_util.py rename to internal/tests/pytests/util/met_util/test_met_util.py index 8241ea4528..7d0df8dd50 100644 --- a/internal_tests/pytests/util/met_util/test_met_util.py +++ b/internal/tests/pytests/util/met_util/test_met_util.py @@ -90,10 +90,10 @@ def test_get_threshold_via_regex(key, value): @pytest.mark.parametrize( 'filename, ext', [ - ('internal_tests/data/zip/testfile.txt', '.gz'), - ('internal_tests/data/zip/testfile2.txt', '.bz2'), - ('internal_tests/data/zip/testfile3.txt', '.zip'), - ('internal_tests/data/zip/testfile4.txt', ''), + ('internal/tests/data/zip/testfile.txt', '.gz'), + ('internal/tests/data/zip/testfile2.txt', '.bz2'), + ('internal/tests/data/zip/testfile3.txt', '.zip'), + ('internal/tests/data/zip/testfile4.txt', ''), ] ) @pytest.mark.util @@ -503,7 +503,7 @@ def test_subset_list(subset_definition, expected_result): def test_get_storm_ids(metplus_config, filename, expected_result): config = metplus_config() filepath = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', + 'internal', 'tests', 'data', 'stat_data', filename) @@ -532,7 +532,7 @@ def test_get_storms(metplus_config, filename, expected_result): storm_id_index = 4 config = metplus_config() filepath = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', + 'internal', 'tests', 'data', 'stat_data', filename) @@ -561,7 +561,7 @@ def test_get_storms_mtd(metplus_config): sort_column = 'OBJECT_CAT' config = metplus_config() filepath = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', + 'internal', 'tests', 'data', 'mtd', 'fake_mtd_2d.txt') diff --git a/internal_tests/pytests/util/metplus_check/test_metplus_check.py b/internal/tests/pytests/util/metplus_check/test_metplus_check.py similarity index 100% rename from internal_tests/pytests/util/metplus_check/test_metplus_check.py rename to internal/tests/pytests/util/metplus_check/test_metplus_check.py diff --git a/internal_tests/pytests/util/string_manip/test_util_string_manip.py b/internal/tests/pytests/util/string_manip/test_util_string_manip.py similarity index 100% rename from internal_tests/pytests/util/string_manip/test_util_string_manip.py rename to internal/tests/pytests/util/string_manip/test_util_string_manip.py diff --git a/internal_tests/pytests/util/string_template_substitution/test_string_template_substitution.py b/internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py similarity index 100% rename from internal_tests/pytests/util/string_template_substitution/test_string_template_substitution.py rename to internal/tests/pytests/util/string_template_substitution/test_string_template_substitution.py diff --git a/internal_tests/pytests/util/time_looping/test_time_looping.py b/internal/tests/pytests/util/time_looping/test_time_looping.py similarity index 100% rename from internal_tests/pytests/util/time_looping/test_time_looping.py rename to internal/tests/pytests/util/time_looping/test_time_looping.py diff --git a/internal_tests/pytests/util/time_util/test_time_util.py b/internal/tests/pytests/util/time_util/test_time_util.py similarity index 100% rename from internal_tests/pytests/util/time_util/test_time_util.py rename to internal/tests/pytests/util/time_util/test_time_util.py diff --git a/internal_tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py b/internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py similarity index 100% rename from internal_tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py rename to internal/tests/pytests/wrappers/ascii2nc/test_ascii2nc_wrapper.py diff --git a/internal_tests/pytests/wrappers/command_builder/test_command_builder.py b/internal/tests/pytests/wrappers/command_builder/test_command_builder.py similarity index 97% rename from internal_tests/pytests/wrappers/command_builder/test_command_builder.py rename to internal/tests/pytests/wrappers/command_builder/test_command_builder.py index 024c53db73..fa95f7e037 100644 --- a/internal_tests/pytests/wrappers/command_builder/test_command_builder.py +++ b/internal/tests/pytests/wrappers/command_builder/test_command_builder.py @@ -9,6 +9,11 @@ from metplus.util import ti_calculate +def get_data_dir(config): + return os.path.join(config.getdir('METPLUS_BASE'), + 'internal', 'tests', 'data', 'obs') + + @pytest.mark.parametrize( 'data_type', [ ("FCST_"), @@ -32,7 +37,7 @@ def test_find_data_no_dated(metplus_config, data_type): pcw.c_dict[f'{data_type}FILE_WINDOW_BEGIN'] = -3600 pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 3600 - pcw.c_dict[f'{data_type}INPUT_DIR'] = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/obs" + pcw.c_dict[f'{data_type}INPUT_DIR'] = get_data_dir(pcw.config) pcw.c_dict[f'{data_type}INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}" obs_file = pcw.find_data(time_info, v, data_type) assert obs_file == pcw.c_dict[f'{data_type}INPUT_DIR']+'/20180201_0045' @@ -80,7 +85,7 @@ def test_find_obs_no_dated(metplus_config): pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600 pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600 - pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir('METPLUS_BASE') + "/internal_tests/data/obs" + pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config) pcw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}" obs_file = pcw.find_obs(time_info, v) assert obs_file == pcw.c_dict['OBS_INPUT_DIR'] + '/20180201_0045' @@ -100,7 +105,7 @@ def test_find_obs_dated(metplus_config): pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600 pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600 - pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/obs" + pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config) pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}' obs_file = pcw.find_obs(time_info, v) assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180201/20180201_0013' @@ -129,7 +134,7 @@ def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds) time_info = ti_calculate(task_info) pcw.c_dict['OFFSETS'] = offsets - pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir('METPLUS_BASE') + "/internal_tests/data/obs" + pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config) pcw.c_dict['OBS_INPUT_TEMPLATE'] = "{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}" obs_file, time_info = pcw.find_obs_offset(time_info, v) @@ -155,7 +160,7 @@ def test_find_obs_dated_previous_day(metplus_config): task_info['lead'] = 0 time_info = ti_calculate(task_info) - pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/obs" + pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config) pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}' pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600 pcw.c_dict['OBS_FILE_WINDOW_END'] = 0 @@ -174,8 +179,8 @@ def test_find_obs_dated_next_day(metplus_config): task_info['valid'] = datetime.datetime.strptime("201802012345", '%Y%m%d%H%M') task_info['lead'] = 0 time_info = ti_calculate(task_info) - - pcw.c_dict['OBS_INPUT_DIR'] = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/obs" + + pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config) pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}' pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = 0 pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600 diff --git a/internal_tests/pytests/wrappers/compare_gridded/test_compare_gridded.py b/internal/tests/pytests/wrappers/compare_gridded/test_compare_gridded.py similarity index 100% rename from internal_tests/pytests/wrappers/compare_gridded/test_compare_gridded.py rename to internal/tests/pytests/wrappers/compare_gridded/test_compare_gridded.py diff --git a/internal_tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py b/internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py similarity index 100% rename from internal_tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py rename to internal/tests/pytests/wrappers/ensemble_stat/test_ensemble_stat_wrapper.py diff --git a/internal_tests/pytests/wrappers/extract_tiles/extract_tiles_test.conf b/internal/tests/pytests/wrappers/extract_tiles/extract_tiles_test.conf similarity index 100% rename from internal_tests/pytests/wrappers/extract_tiles/extract_tiles_test.conf rename to internal/tests/pytests/wrappers/extract_tiles/extract_tiles_test.conf diff --git a/internal_tests/pytests/wrappers/extract_tiles/test_extract_tiles.py b/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py similarity index 95% rename from internal_tests/pytests/wrappers/extract_tiles/test_extract_tiles.py rename to internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py index f4b1f1ed6f..41d54886f0 100644 --- a/internal_tests/pytests/wrappers/extract_tiles/test_extract_tiles.py +++ b/internal/tests/pytests/wrappers/extract_tiles/test_extract_tiles.py @@ -25,16 +25,16 @@ def extract_tiles_wrapper(metplus_config): def get_storm_lines(wrapper): filter_file = os.path.join(wrapper.config.getdir('METPLUS_BASE'), - 'internal_tests', - 'data', - 'stat_data', - 'fake_filter_20141214_00.tcst') + 'internal', 'tests', + 'data', + 'stat_data', + 'fake_filter_20141214_00.tcst') return get_input_lines(filter_file) def get_mtd_lines(wrapper): input_file = os.path.join(wrapper.config.getdir('METPLUS_BASE'), - 'internal_tests', + 'internal', 'tests', 'data', 'mtd', 'fake_mtd_2d.txt') diff --git a/internal_tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py similarity index 99% rename from internal_tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py rename to internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py index e793ba27b1..f8ed2b4639 100644 --- a/internal_tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py +++ b/internal/tests/pytests/wrappers/gen_ens_prod/test_gen_ens_prod_wrapper.py @@ -47,7 +47,7 @@ def set_minimum_config_settings(config): def handle_input_dir(config): test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', + 'internal', 'tests', 'data') input_dir = os.path.join(test_data_dir, 'ens') config.set('config', 'GEN_ENS_PROD_INPUT_DIR', input_dir) diff --git a/internal_tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py b/internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py similarity index 100% rename from internal_tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py rename to internal/tests/pytests/wrappers/gen_vx_mask/test_gen_vx_mask.py diff --git a/internal_tests/pytests/wrappers/grid_diag/test_grid_diag.py b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py similarity index 99% rename from internal_tests/pytests/wrappers/grid_diag/test_grid_diag.py rename to internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py index 3c9178229c..61fd599150 100644 --- a/internal_tests/pytests/wrappers/grid_diag/test_grid_diag.py +++ b/internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py @@ -78,7 +78,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expected_subset): config.set('config', 'LEAD_SEQ', '0H, 24H') input_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', + 'internal', 'tests', 'data', 'user_script') config.set('config', 'GRID_DIAG_INPUT_DIR', input_dir) diff --git a/internal_tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py b/internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py similarity index 100% rename from internal_tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py rename to internal/tests/pytests/wrappers/grid_stat/test_grid_stat_wrapper.py diff --git a/internal_tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py b/internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py similarity index 100% rename from internal_tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py rename to internal/tests/pytests/wrappers/ioda2nc/test_ioda2nc_wrapper.py diff --git a/internal_tests/pytests/wrappers/met_db_load/test_met_db_load.py b/internal/tests/pytests/wrappers/met_db_load/test_met_db_load.py similarity index 100% rename from internal_tests/pytests/wrappers/met_db_load/test_met_db_load.py rename to internal/tests/pytests/wrappers/met_db_load/test_met_db_load.py diff --git a/internal_tests/pytests/wrappers/mode/test_mode_wrapper.py b/internal/tests/pytests/wrappers/mode/test_mode_wrapper.py similarity index 100% rename from internal_tests/pytests/wrappers/mode/test_mode_wrapper.py rename to internal/tests/pytests/wrappers/mode/test_mode_wrapper.py diff --git a/internal_tests/pytests/wrappers/mtd/test_mtd_wrapper.py b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py similarity index 93% rename from internal_tests/pytests/wrappers/mtd/test_mtd_wrapper.py rename to internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py index 524741b0d6..135c62031f 100644 --- a/internal_tests/pytests/wrappers/mtd/test_mtd_wrapper.py +++ b/internal/tests/pytests/wrappers/mtd/test_mtd_wrapper.py @@ -8,6 +8,11 @@ from metplus.wrappers.mtd_wrapper import MTDWrapper +def get_test_data_dir(config, subdir): + return os.path.join(config.getdir('METPLUS_BASE'), + 'internal', 'tests', 'data', subdir) + + def mtd_wrapper(metplus_config, lead_seq=None): """! Returns a default MTDWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration @@ -30,8 +35,8 @@ def mtd_wrapper(metplus_config, lead_seq=None): @pytest.mark.wrapper def test_mtd_by_init_all_found(metplus_config): mw = mtd_wrapper(metplus_config, '1,2,3') - obs_dir = mw.config.getdir('METPLUS_BASE')+"/internal_tests/data/obs" - fcst_dir = mw.config.getdir('METPLUS_BASE')+"/internal_tests/data/fcst" + obs_dir = get_test_data_dir(mw.config, 'obs') + fcst_dir = get_test_data_dir(mw.config, 'fcst') mw.c_dict['OBS_INPUT_DIR'] = obs_dir mw.c_dict['FCST_INPUT_DIR'] = fcst_dir mw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A{level?fmt=%.2H}.nc" @@ -64,8 +69,8 @@ def test_mtd_by_init_all_found(metplus_config): @pytest.mark.wrapper def test_mtd_by_valid_all_found(metplus_config): mw = mtd_wrapper(metplus_config, '1, 2, 3') - obs_dir = mw.config.getdir('METPLUS_BASE')+"/internal_tests/data/obs" - fcst_dir = mw.config.getdir('METPLUS_BASE')+"/internal_tests/data/fcst" + obs_dir = get_test_data_dir(mw.config, 'obs') + fcst_dir = get_test_data_dir(mw.config, 'fcst') mw.c_dict['OBS_INPUT_DIR'] = obs_dir mw.c_dict['FCST_INPUT_DIR'] = fcst_dir mw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A{level?fmt=%.2H}.nc" @@ -98,8 +103,8 @@ def test_mtd_by_valid_all_found(metplus_config): @pytest.mark.wrapper def test_mtd_by_init_miss_fcst(metplus_config): mw = mtd_wrapper(metplus_config, '3, 6, 9, 12') - obs_dir = mw.config.getdir('METPLUS_BASE')+"/internal_tests/data/obs" - fcst_dir = mw.config.getdir('METPLUS_BASE')+"/internal_tests/data/fcst" + obs_dir = get_test_data_dir(mw.config, 'obs') + fcst_dir = get_test_data_dir(mw.config, 'fcst') mw.c_dict['OBS_INPUT_DIR'] = obs_dir mw.c_dict['FCST_INPUT_DIR'] = fcst_dir mw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A{level?fmt=%.2H}.nc" @@ -132,8 +137,8 @@ def test_mtd_by_init_miss_fcst(metplus_config): @pytest.mark.wrapper def test_mtd_by_init_miss_both(metplus_config): mw = mtd_wrapper(metplus_config, '6, 12, 18') - obs_dir = mw.config.getdir('METPLUS_BASE')+"/internal_tests/data/obs" - fcst_dir = mw.config.getdir('METPLUS_BASE')+"/internal_tests/data/fcst" + obs_dir = get_test_data_dir(mw.config, 'obs') + fcst_dir = get_test_data_dir(mw.config, 'fcst') mw.c_dict['OBS_INPUT_DIR'] = obs_dir mw.c_dict['FCST_INPUT_DIR'] = fcst_dir mw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}/qpe_{valid?fmt=%Y%m%d%H}_A{level?fmt=%.2H}.nc" @@ -164,7 +169,7 @@ def test_mtd_by_init_miss_both(metplus_config): @pytest.mark.wrapper def test_mtd_single(metplus_config): mw = mtd_wrapper(metplus_config, '1, 2, 3') - fcst_dir = mw.config.getdir('METPLUS_BASE')+"/internal_tests/data/fcst" + fcst_dir = get_test_data_dir(mw.config, 'fcst') mw.c_dict['SINGLE_RUN'] = True mw.c_dict['SINGLE_DATA_SRC'] = 'FCST' mw.c_dict['FCST_INPUT_DIR'] = fcst_dir diff --git a/internal_tests/pytests/wrappers/pb2nc/__init__.py b/internal/tests/pytests/wrappers/pb2nc/__init__.py similarity index 100% rename from internal_tests/pytests/wrappers/pb2nc/__init__.py rename to internal/tests/pytests/wrappers/pb2nc/__init__.py diff --git a/internal_tests/pytests/wrappers/pb2nc/conf1 b/internal/tests/pytests/wrappers/pb2nc/conf1 similarity index 100% rename from internal_tests/pytests/wrappers/pb2nc/conf1 rename to internal/tests/pytests/wrappers/pb2nc/conf1 diff --git a/internal_tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py b/internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py similarity index 100% rename from internal_tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py rename to internal/tests/pytests/wrappers/pb2nc/test_pb2nc_wrapper.py diff --git a/internal_tests/pytests/wrappers/pcp_combine/test1.conf b/internal/tests/pytests/wrappers/pcp_combine/test1.conf similarity index 79% rename from internal_tests/pytests/wrappers/pcp_combine/test1.conf rename to internal/tests/pytests/wrappers/pcp_combine/test1.conf index 1fd7a2d7bd..0d50280991 100644 --- a/internal_tests/pytests/wrappers/pcp_combine/test1.conf +++ b/internal/tests/pytests/wrappers/pcp_combine/test1.conf @@ -21,11 +21,11 @@ OBS_PCP_COMBINE_RUN = True OBS_PCP_COMBINE_METHOD = ADD [dir] -OBS_PCP_COMBINE_INPUT_DIR = {METPLUS_BASE}/internal_tests/data/accum -OBS_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/internal_tests/data/fakeout +OBS_PCP_COMBINE_INPUT_DIR = {METPLUS_BASE}/internal/tests/data/accum +OBS_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/internal/tests/data/fakeout -FCST_PCP_COMBINE_INPUT_DIR = {METPLUS_BASE}/internal_tests/data/fcst -FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/internal_tests/data/fakeout +FCST_PCP_COMBINE_INPUT_DIR = {METPLUS_BASE}/internal/tests/data/fcst +FCST_PCP_COMBINE_OUTPUT_DIR = {OUTPUT_BASE}/internal/tests/data/fakeout [filename_templates] OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}.{level?fmt=%HH}h diff --git a/internal_tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py b/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py similarity index 95% rename from internal_tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py rename to internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py index 603d4c0943..725278c393 100644 --- a/internal_tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py +++ b/internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py @@ -9,6 +9,13 @@ from metplus.util import ti_calculate +def get_test_data_dir(config, subdir=None): + top_dir = os.path.join(config.getdir('METPLUS_BASE'), + 'internal', 'tests', 'data') + if subdir: + top_dir = os.path.join(top_dir, subdir) + return top_dir + def pcp_combine_wrapper(metplus_config, d_type): """! Returns a default PCPCombineWrapper with /path/to entries in the metplus_system.conf and metplus_runtime.conf configuration @@ -32,7 +39,7 @@ def pcp_combine_wrapper(metplus_config, d_type): def test_get_accumulation_1_to_6(metplus_config): data_src = "OBS" pcw = pcp_combine_wrapper(metplus_config, data_src) - input_dir = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/accum" + input_dir = get_test_data_dir(pcw.config, subdir='accum') task_info = {} task_info['valid'] = datetime.strptime("2016090418", '%Y%m%d%H') time_info = ti_calculate(task_info) @@ -57,7 +64,7 @@ def test_get_accumulation_1_to_6(metplus_config): def test_get_accumulation_6_to_6(metplus_config): data_src = "FCST" pcw = pcp_combine_wrapper(metplus_config, data_src) - input_dir = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/accum" + input_dir = get_test_data_dir(pcw.config, subdir='accum') task_info = {} task_info['valid'] = datetime.strptime("2016090418", '%Y%m%d%H') time_info = ti_calculate(task_info) @@ -79,7 +86,7 @@ def test_get_accumulation_6_to_6(metplus_config): def test_get_lowest_forecast_file_dated_subdir(metplus_config): data_src = "FCST" pcw = pcp_combine_wrapper(metplus_config, data_src) - input_dir = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/fcst" + input_dir = get_test_data_dir(pcw.config, subdir='fcst') valid_time = datetime.strptime("201802012100", '%Y%m%d%H%M') pcw.c_dict[f'{data_src}_INPUT_DIR'] = input_dir pcw._build_input_accum_list(data_src, {'valid': valid_time}) @@ -93,7 +100,7 @@ def test_forecast_constant_init(metplus_config): data_src = "FCST" pcw = pcp_combine_wrapper(metplus_config, data_src) pcw.c_dict['FCST_CONSTANT_INIT'] = True - input_dir = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/fcst" + input_dir = get_test_data_dir(pcw.config, subdir='fcst') init_time = datetime.strptime("2018020112", '%Y%m%d%H') valid_time = datetime.strptime("2018020121", '%Y%m%d%H') pcw.c_dict[f'{data_src}_INPUT_DIR'] = input_dir @@ -107,7 +114,7 @@ def test_forecast_not_constant_init(metplus_config): data_src = "FCST" pcw = pcp_combine_wrapper(metplus_config, data_src) pcw.c_dict['FCST_CONSTANT_INIT'] = False - input_dir = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/fcst" + input_dir = get_test_data_dir(pcw.config, subdir='fcst') init_time = datetime.strptime("2018020112", '%Y%m%d%H') valid_time = datetime.strptime("2018020121", '%Y%m%d%H') pcw.c_dict[f'{data_src}_INPUT_DIR'] = input_dir @@ -121,7 +128,7 @@ def test_forecast_not_constant_init(metplus_config): def test_get_lowest_forecast_file_no_subdir(metplus_config): data_src = "FCST" pcw = pcp_combine_wrapper(metplus_config, data_src) - input_dir = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/fcst" + input_dir = get_test_data_dir(pcw.config, subdir='fcst') valid_time = datetime.strptime("201802012100", '%Y%m%d%H%M') template = "file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc" pcw.c_dict[f'{data_src}_INPUT_TEMPLATE'] = template @@ -135,7 +142,7 @@ def test_get_lowest_forecast_file_no_subdir(metplus_config): def test_get_lowest_forecast_file_yesterday(metplus_config): data_src = "FCST" pcw = pcp_combine_wrapper(metplus_config, data_src) - input_dir = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/fcst" + input_dir = get_test_data_dir(pcw.config, subdir='fcst') valid_time = datetime.strptime("201802010600", '%Y%m%d%H%M') template = "file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc" pcw.c_dict[f'{data_src}_INPUT_TEMPLATE'] = template @@ -153,7 +160,7 @@ def test_setup_add_method(metplus_config): task_info['valid'] = datetime.strptime("2016090418", '%Y%m%d%H') time_info = ti_calculate(task_info) - input_dir = pcw.config.getdir('METPLUS_BASE')+"/internal_tests/data/accum" + input_dir = get_test_data_dir(pcw.config, subdir='accum') lookback = 6 * 3600 files_found = pcw.setup_add_method(time_info, lookback, data_src) assert files_found @@ -205,9 +212,7 @@ def test_pcp_combine_add_subhourly(metplus_config): fcst_fmt = f'\'name="{fcst_name}"; level="{fcst_level}";\'' config = metplus_config() - test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', - 'data') + test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, 'pcp_in', 'add') @@ -272,9 +277,7 @@ def test_pcp_combine_bucket(metplus_config): fcst_output_name = 'APCP' config = metplus_config() - test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', - 'data') + test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, 'pcp_in', 'bucket') @@ -354,9 +357,7 @@ def test_pcp_combine_derive(metplus_config, config_overrides, extra_fields): fcst_fmt = f'-field \'name="{fcst_name}"; level="{fcst_level}";\'' config = metplus_config() - test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', - 'data') + test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, 'pcp_in', 'derive') @@ -429,9 +430,7 @@ def test_pcp_combine_loop_custom(metplus_config): ens_list = ['ens1', 'ens2', 'ens3', 'ens4', 'ens5', 'ens6'] config = metplus_config() - test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', - 'data') + test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, 'pcp_in', 'loop_custom') @@ -493,9 +492,7 @@ def test_pcp_combine_loop_custom(metplus_config): def test_pcp_combine_subtract(metplus_config): config = metplus_config() - test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', - 'data') + test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, 'pcp_in', 'derive') @@ -558,9 +555,7 @@ def test_pcp_combine_sum_subhourly(metplus_config): fcst_fmt = f'-field \'name="{fcst_name}"; level="{fcst_level}";\'' config = metplus_config() - test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', - 'data') + test_data_dir = get_test_data_dir(config) fcst_input_dir = os.path.join(test_data_dir, 'pcp_in', 'add') diff --git a/internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py b/internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py similarity index 100% rename from internal_tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py rename to internal/tests/pytests/wrappers/plot_point_obs/test_plot_point_obs_wrapper.py diff --git a/internal_tests/pytests/wrappers/point2grid/test_point2grid.py b/internal/tests/pytests/wrappers/point2grid/test_point2grid.py similarity index 100% rename from internal_tests/pytests/wrappers/point2grid/test_point2grid.py rename to internal/tests/pytests/wrappers/point2grid/test_point2grid.py diff --git a/internal_tests/pytests/wrappers/point_stat/__init__.py b/internal/tests/pytests/wrappers/point_stat/__init__.py similarity index 100% rename from internal_tests/pytests/wrappers/point_stat/__init__.py rename to internal/tests/pytests/wrappers/point_stat/__init__.py diff --git a/internal_tests/pytests/wrappers/point_stat/point_stat_test_conus_sfc.conf b/internal/tests/pytests/wrappers/point_stat/point_stat_test_conus_sfc.conf similarity index 100% rename from internal_tests/pytests/wrappers/point_stat/point_stat_test_conus_sfc.conf rename to internal/tests/pytests/wrappers/point_stat/point_stat_test_conus_sfc.conf diff --git a/internal_tests/pytests/wrappers/point_stat/point_stat_test_upper_air.conf b/internal/tests/pytests/wrappers/point_stat/point_stat_test_upper_air.conf similarity index 100% rename from internal_tests/pytests/wrappers/point_stat/point_stat_test_upper_air.conf rename to internal/tests/pytests/wrappers/point_stat/point_stat_test_upper_air.conf diff --git a/internal_tests/pytests/wrappers/point_stat/refactor_point_stat_test_conus_sfc.conf b/internal/tests/pytests/wrappers/point_stat/refactor_point_stat_test_conus_sfc.conf similarity index 100% rename from internal_tests/pytests/wrappers/point_stat/refactor_point_stat_test_conus_sfc.conf rename to internal/tests/pytests/wrappers/point_stat/refactor_point_stat_test_conus_sfc.conf diff --git a/internal_tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py b/internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py similarity index 100% rename from internal_tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py rename to internal/tests/pytests/wrappers/point_stat/test_point_stat_wrapper.py diff --git a/internal_tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py b/internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py similarity index 100% rename from internal_tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py rename to internal/tests/pytests/wrappers/regrid_data_plane/test_regrid_data_plane.py diff --git a/internal_tests/pytests/wrappers/runtime_freq/test_runtime_freq.py b/internal/tests/pytests/wrappers/runtime_freq/test_runtime_freq.py similarity index 100% rename from internal_tests/pytests/wrappers/runtime_freq/test_runtime_freq.py rename to internal/tests/pytests/wrappers/runtime_freq/test_runtime_freq.py diff --git a/internal_tests/pytests/wrappers/series_analysis/series_test.conf b/internal/tests/pytests/wrappers/series_analysis/series_test.conf similarity index 100% rename from internal_tests/pytests/wrappers/series_analysis/series_test.conf rename to internal/tests/pytests/wrappers/series_analysis/series_test.conf diff --git a/internal_tests/pytests/wrappers/series_analysis/test_series_analysis.py b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py similarity index 99% rename from internal_tests/pytests/wrappers/series_analysis/test_series_analysis.py rename to internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py index ca534272c0..5259e03ef1 100644 --- a/internal_tests/pytests/wrappers/series_analysis/test_series_analysis.py +++ b/internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py @@ -27,7 +27,7 @@ def get_input_dirs(config): fake_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', + 'internal', 'tests', 'data') stat_input_dir = os.path.join(fake_data_dir, 'stat_data') @@ -367,7 +367,7 @@ def test_get_fcst_file_info(metplus_config): output_dir = ( os.path.join(wrapper.config.getdir('METPLUS_BASE'), - 'internal_tests', + 'internal', 'tests', 'data', 'file_lists') ) diff --git a/internal_tests/pytests/wrappers/stat_analysis/test.conf b/internal/tests/pytests/wrappers/stat_analysis/test.conf similarity index 97% rename from internal_tests/pytests/wrappers/stat_analysis/test.conf rename to internal/tests/pytests/wrappers/stat_analysis/test.conf index d07726f2f4..de84a88976 100644 --- a/internal_tests/pytests/wrappers/stat_analysis/test.conf +++ b/internal/tests/pytests/wrappers/stat_analysis/test.conf @@ -2,9 +2,9 @@ # Output data directory STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE}/stat_analysis # Location of configuration files used by MET applications -CONFIG_DIR = {METPLUS_BASE}/internal_tests/config +CONFIG_DIR = {METPLUS_BASE}/internal/tests/config -MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {METPLUS_BASE}/internal_tests/data/stat_data +MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {METPLUS_BASE}/internal/tests/data/stat_data [config] # Options are times, processes diff --git a/internal_tests/pytests/wrappers/stat_analysis/test_plotting.conf b/internal/tests/pytests/wrappers/stat_analysis/test_plotting.conf similarity index 98% rename from internal_tests/pytests/wrappers/stat_analysis/test_plotting.conf rename to internal/tests/pytests/wrappers/stat_analysis/test_plotting.conf index 284df8c546..d0b462f43a 100644 --- a/internal_tests/pytests/wrappers/stat_analysis/test_plotting.conf +++ b/internal/tests/pytests/wrappers/stat_analysis/test_plotting.conf @@ -6,7 +6,7 @@ MAKE_PLOTS_SCRIPTS_DIR = {METPLUS_BASE}/ush/plotting_scripts MAKE_PLOTS_INPUT_DIR = {STAT_ANALYSIS_OUTPUT_DIR} MAKE_PLOTS_OUTPUT_DIR = {OUTPUT_BASE}/plotting/make_plots # Location of configuration files used by MET applications -CONFIG_DIR = {METPLUS_BASE}/internal_tests/plotting/met_config +CONFIG_DIR = {METPLUS_BASE}/internal/tests/plotting/met_config MODEL1_STAT_ANALYSIS_LOOKIN_DIR = {OUTPUT_BASE}/stat_analysis MODEL2_STAT_ANALYSIS_LOOKIN_DIR = {OUTPUT_BASE}/stat_analysis diff --git a/internal_tests/pytests/wrappers/stat_analysis/test_stat_analysis.py b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py similarity index 99% rename from internal_tests/pytests/wrappers/stat_analysis/test_stat_analysis.py rename to internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py index f076feeb6f..054e167bc5 100644 --- a/internal_tests/pytests/wrappers/stat_analysis/test_stat_analysis.py +++ b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis.py @@ -8,7 +8,7 @@ from metplus.wrappers.stat_analysis_wrapper import StatAnalysisWrapper from metplus.util import handle_tmp_dir -METPLUS_BASE = os.getcwd().split('/internal_tests')[0] +METPLUS_BASE = os.getcwd().split('/internal')[0] def stat_analysis_wrapper(metplus_config): @@ -55,7 +55,7 @@ def test_create_c_dict(metplus_config): # Test 1 c_dict = st.create_c_dict() assert c_dict['LOOP_ORDER'] == 'times' - assert(os.path.realpath(c_dict['CONFIG_FILE']) == (METPLUS_BASE+'/internal_tests/' + assert(os.path.realpath(c_dict['CONFIG_FILE']) == (METPLUS_BASE+'/internal/tests/' +'config/STATAnalysisConfig')) assert(c_dict['OUTPUT_DIR'] == (st.config.getdir('OUTPUT_BASE') +'/stat_analysis')) @@ -728,7 +728,7 @@ def test_run_stat_analysis(metplus_config): # Test 1 expected_filename = (st.config.getdir('OUTPUT_BASE')+'/stat_analysis' +'/00Z/MODEL_TEST/MODEL_TEST_20190101.stat') - comparison_filename = (METPLUS_BASE+'/internal_tests/data/stat_data/' + comparison_filename = (METPLUS_BASE+'/internal/tests/data/stat_data/' +'test_20190101.stat') st.c_dict['DATE_BEG'] = '20190101' st.c_dict['DATE_END'] = '20190101' diff --git a/internal_tests/pytests/wrappers/stat_analysis/test_stat_analysis_plotting.py b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis_plotting.py similarity index 99% rename from internal_tests/pytests/wrappers/stat_analysis/test_stat_analysis_plotting.py rename to internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis_plotting.py index e869b5d580..c7452295ec 100644 --- a/internal_tests/pytests/wrappers/stat_analysis/test_stat_analysis_plotting.py +++ b/internal/tests/pytests/wrappers/stat_analysis/test_stat_analysis_plotting.py @@ -9,7 +9,7 @@ from metplus.wrappers.stat_analysis_wrapper import StatAnalysisWrapper from metplus.util import handle_tmp_dir -METPLUS_BASE = os.getcwd().split('/internal_tests')[0] +METPLUS_BASE = os.getcwd().split('/internal')[0] def stat_analysis_wrapper(metplus_config): diff --git a/internal_tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py similarity index 99% rename from internal_tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py rename to internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py index d424abbeae..248228fa92 100644 --- a/internal_tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_gen/test_tc_gen_wrapper.py @@ -296,7 +296,7 @@ def test_tc_gen(metplus_config, config_overrides, env_var_values): config = metplus_config() test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', + 'internal', 'tests', 'data', 'tc_gen') track_dir = os.path.join(test_data_dir, 'track') diff --git a/internal_tests/pytests/wrappers/tc_pairs/tc_pairs_wrapper_test.conf b/internal/tests/pytests/wrappers/tc_pairs/tc_pairs_wrapper_test.conf similarity index 100% rename from internal_tests/pytests/wrappers/tc_pairs/tc_pairs_wrapper_test.conf rename to internal/tests/pytests/wrappers/tc_pairs/tc_pairs_wrapper_test.conf diff --git a/internal_tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py similarity index 96% rename from internal_tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py rename to internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py index bea68e7492..8f9de6d6ff 100644 --- a/internal_tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py +++ b/internal/tests/pytests/wrappers/tc_pairs/test_tc_pairs_wrapper.py @@ -17,6 +17,11 @@ run_times = ['2014121318'] +def get_data_dir(config): + return os.path.join(config.getdir('METPLUS_BASE'), + 'internal', 'tests', 'data', 'tc_pairs') + + def set_minimum_config_settings(config, loop_by='INIT'): # set config variables to prevent command from running and bypass check # if input files actually exist @@ -140,10 +145,7 @@ def test_get_bdeck(metplus_config, basin, cyclone, expected_files, set_minimum_config_settings(config) - test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', - 'data', - 'tc_pairs') + test_data_dir = get_data_dir(config) bdeck_dir = os.path.join(test_data_dir, 'bdeck') config.set('config', 'TC_PAIRS_BDECK_INPUT_DIR', bdeck_dir) @@ -252,10 +254,7 @@ def test_tc_pairs_storm_id_lists(metplus_config, config_overrides, config.set('config', 'INIT_BEG', '2019') config.set('config', 'INIT_END', '2019') - test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', - 'data', - 'tc_pairs') + test_data_dir = get_data_dir(config) bdeck_dir = os.path.join(test_data_dir, 'bdeck') edeck_dir = os.path.join(test_data_dir, 'edeck') @@ -394,10 +393,7 @@ def test_tc_pairs_loop_order_processes(metplus_config, config_overrides, set_minimum_config_settings(config, loop_by) - test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', - 'data', - 'tc_pairs') + test_data_dir = get_data_dir(config) bdeck_dir = os.path.join(test_data_dir, 'bdeck') adeck_dir = os.path.join(test_data_dir, 'adeck') @@ -493,10 +489,7 @@ def test_tc_pairs_read_all_files(metplus_config, config_overrides, set_minimum_config_settings(config, loop_by) - test_data_dir = os.path.join(config.getdir('METPLUS_BASE'), - 'internal_tests', - 'data', - 'tc_pairs') + test_data_dir = get_data_dir(config) bdeck_dir = os.path.join(test_data_dir, 'bdeck') adeck_dir = os.path.join(test_data_dir, 'adeck') diff --git a/internal_tests/pytests/wrappers/tc_stat/tc_stat_conf.conf b/internal/tests/pytests/wrappers/tc_stat/tc_stat_conf.conf similarity index 100% rename from internal_tests/pytests/wrappers/tc_stat/tc_stat_conf.conf rename to internal/tests/pytests/wrappers/tc_stat/tc_stat_conf.conf diff --git a/internal_tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py b/internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py similarity index 100% rename from internal_tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py rename to internal/tests/pytests/wrappers/tc_stat/test_tc_stat_wrapper.py diff --git a/internal_tests/pytests/wrappers/user_script/test_user_script.py b/internal/tests/pytests/wrappers/user_script/test_user_script.py similarity index 100% rename from internal_tests/pytests/wrappers/user_script/test_user_script.py rename to internal/tests/pytests/wrappers/user_script/test_user_script.py diff --git a/internal_tests/use_cases/all_use_cases.txt b/internal/tests/use_cases/all_use_cases.txt similarity index 100% rename from internal_tests/use_cases/all_use_cases.txt rename to internal/tests/use_cases/all_use_cases.txt diff --git a/internal_tests/use_cases/metplus_test_env.corrinado.sh b/internal/tests/use_cases/metplus_test_env.corrinado.sh similarity index 100% rename from internal_tests/use_cases/metplus_test_env.corrinado.sh rename to internal/tests/use_cases/metplus_test_env.corrinado.sh diff --git a/internal_tests/use_cases/metplus_test_env.dakota.sh b/internal/tests/use_cases/metplus_test_env.dakota.sh similarity index 100% rename from internal_tests/use_cases/metplus_test_env.dakota.sh rename to internal/tests/use_cases/metplus_test_env.dakota.sh diff --git a/internal_tests/use_cases/metplus_test_env.docker.sh b/internal/tests/use_cases/metplus_test_env.docker.sh similarity index 100% rename from internal_tests/use_cases/metplus_test_env.docker.sh rename to internal/tests/use_cases/metplus_test_env.docker.sh diff --git a/internal_tests/use_cases/metplus_test_env.eyewall.sh b/internal/tests/use_cases/metplus_test_env.eyewall.sh similarity index 100% rename from internal_tests/use_cases/metplus_test_env.eyewall.sh rename to internal/tests/use_cases/metplus_test_env.eyewall.sh diff --git a/internal_tests/use_cases/metplus_test_env.kiowa.sh b/internal/tests/use_cases/metplus_test_env.kiowa.sh similarity index 100% rename from internal_tests/use_cases/metplus_test_env.kiowa.sh rename to internal/tests/use_cases/metplus_test_env.kiowa.sh diff --git a/internal_tests/use_cases/metplus_test_env.venus.sh b/internal/tests/use_cases/metplus_test_env.venus.sh similarity index 100% rename from internal_tests/use_cases/metplus_test_env.venus.sh rename to internal/tests/use_cases/metplus_test_env.venus.sh diff --git a/internal_tests/use_cases/metplus_use_case_suite.py b/internal/tests/use_cases/metplus_use_case_suite.py similarity index 100% rename from internal_tests/use_cases/metplus_use_case_suite.py rename to internal/tests/use_cases/metplus_use_case_suite.py diff --git a/internal_tests/use_cases/run_test_use_cases.sh b/internal/tests/use_cases/run_test_use_cases.sh similarity index 100% rename from internal_tests/use_cases/run_test_use_cases.sh rename to internal/tests/use_cases/run_test_use_cases.sh diff --git a/internal_tests/use_cases/setup_next_release_data.py b/internal/tests/use_cases/setup_next_release_data.py similarity index 100% rename from internal_tests/use_cases/setup_next_release_data.py rename to internal/tests/use_cases/setup_next_release_data.py diff --git a/internal_tests/use_cases/system.conf b/internal/tests/use_cases/system.conf similarity index 100% rename from internal_tests/use_cases/system.conf rename to internal/tests/use_cases/system.conf diff --git a/internal_tests/use_cases/test_use_cases.py b/internal/tests/use_cases/test_use_cases.py similarity index 99% rename from internal_tests/use_cases/test_use_cases.py rename to internal/tests/use_cases/test_use_cases.py index 0f62f41645..ddef88a9e2 100644 --- a/internal_tests/use_cases/test_use_cases.py +++ b/internal/tests/use_cases/test_use_cases.py @@ -34,7 +34,7 @@ use_case_dir = os.path.join(metplus_home, "parm/use_cases") def get_param_list(param): - conf = metplus_home+"/internal_tests/use_cases/system.conf" + conf = metplus_home+"/internal/tests/use_cases/system.conf" params = param.split(",") params = params + [conf] return params diff --git a/internal_tests/METplus_conf_util.py b/internal_tests/METplus_conf_util.py deleted file mode 100755 index b9cbfe3e03..0000000000 --- a/internal_tests/METplus_conf_util.py +++ /dev/null @@ -1,250 +0,0 @@ -#!/usr/local/python/bin/python -# -# File: METplus_conf_util.py -# -# Author: D. Adriaansen -# -# Date: 01 Jun 2018 -# -# Purpose: Distill down all of the configuration options from conf files in the METplus repository, -# remove duplicates, and check against an existing file to see if there are new/removed conf -# options. Also provide an option for the user to replace the current existing file with an -# updated one. This is mainly useful for keeping documentation up to date, as a developer utility -# to make sure if they add new conf options, they are captured and the documentation can be updated -# before pushing changes. -# -# Notes: -#______________________________________________________________________________________________________ - -# Python libraries -import string, subprocess, sys, os - -##################################### User Config ############################################# - -# Path to METplus install -METPLUS_BASE = os.path.expanduser("~/projects/METplus") - -# Path to existing static conf list. If this is the first time running, -# then set this to where you want to write the static conf list file and -# use mode "static" (see usage) -STATIC_CONF_LIST = "%s/internal_tests/conf_list.static" % (METPLUS_BASE) - -# Path to temp directory to write a file to compare against -TMP_CONF_LIST_DIR = "/tmp" - -# Debug flag -DEBUG = True - -# List of files to check for config items -chklist = ['%s/parm/metplus_config/*.conf' % (METPLUS_BASE),'%s/parm/use_cases/*/*.conf' % (METPLUS_BASE), \ - '%s/parm/use_cases/*/*/*.conf' % (METPLUS_BASE)] - -# List for current conf items -clist = [] - -# List for temp conf items -tlist = [] - -############################################################################################### - -# Define usage statement for user -def usage(): - print("") - print(" USAGE: METplus_conf_util.py ") - print("") - print(" Where is either:") - print(" check -or- update -or- static") - print(" check = print list of differences") - print(" update = same as check, but replace existing conf list with new one as well") - print(" static = generate a single file from the current repository branch and don't compare") - print("") - print(" EXITING...") - print("") - sys.exit(1) - -# Function to execute shell commands -def call(cmd): - if DEBUG: - print(" "+cmd) - subprocess.call('%s' % (cmd), shell=True, executable='/bin/csh') - -# Check users' arguments -if len(sys.argv)!=2 or sys.argv[1] == "-h" or sys.argv[1] == "--help": - usage() -else: - mode = sys.argv[1] - -# Based on the requested mode, print things for the user -if mode=='update': - print("") - print(" WARNING! YOU ARE ABOUT TO OVERWRITE ANY EXISTING STATIC CONF LIST LOCATED HERE:") - print(" "+STATIC_CONF_LIST) - print("") - print(" IS THIS WHAT YOU WANT TO DO?") - status = raw_input(" y/n: ") - if status == 'n': - print("") - print(" EXITING...") - print("") - sys.exit(1) - else: - print("") - print(" UPDATING STATIC CONF LIST...") -elif mode=='check': - print("") - print(" PREPARING LIST OF DIFFERENCES BETWEEN REPOSITORY AND EXISTING CONF LIST...") -elif mode=='static': - print("") - print(" WARNING! YOU ARE ABOUT TO GENERATE A NEW CONF LIST AND OVERWRITE ANY EXISTING CONF LIST LOCATED HERE:") - print(" "+STATIC_CONF_LIST) - print("") - print(" *** THIS MODE IS TYPICALLY NOT NEEDED! ***") - print("") - print(" IS THIS REALLY WHAT YOU WANT TO DO?") - status = raw_input(" y/n: ") - if status=='n': - print("") - print(" EXITING...") - print("") - sys.exit(1) - else: - print("") - print(" WRITING STATIC CONF LIST TO %s..." % (STATIC_CONF_LIST)) -else: - print("") - print(" FATAL! UNKNOWN MODE.") - usage() - -# Run the grep commands to create the temporary files. This is needed regardless of what the user is requesting. -if DEBUG: - print("") - print(" GATHERING CONF ITEMS FROM:") - for f in chklist: - print(" "+f) - print("") -cmd = "cat " -for f in chklist: - cmd += f+" " -cmd += "> %s/tmp.conf" % (TMP_CONF_LIST_DIR) -call(cmd) -cmd = "grep = %s/tmp.conf | grep -v \# | sort | uniq > %s/unique.conf" % (TMP_CONF_LIST_DIR,TMP_CONF_LIST_DIR) -call(cmd) - -############ For static mode, just write out a file and exit. -if mode=='static': - # Open temp file and read - tfile = open("%s/unique.conf" % (TMP_CONF_LIST_DIR)) - tdata = tfile.readlines() - tfile.close() - - # Create a list of unique items - for tl in tdata: - s1 = string.split(tl,"=") - tlist.append(s1[0].strip()) - set(tlist) - tlist = list(set(tlist)) - tlist.sort() - - # Open file for writing - sfout = open(STATIC_CONF_LIST,"w") - for i in tlist: - sfout.write("%s\n" % (i)) - - # Print entries found - print("") - print(" TOTAL CONF ENTRIES FOUND: %04d" % (len(tlist))) - - print("") - print(" DONE.") - sfout.close() - sys.exit(0) - -########### For check and update modes, we need all the information below. -# Open current file and read -cfile = open(STATIC_CONF_LIST,"r") -cdata = cfile.readlines() -cfile.close() - -# Open temp file and read -tfile = open("%s/unique.conf" % (TMP_CONF_LIST_DIR)) -tdata = tfile.readlines() -tfile.close() - -# Create lists of unique items -for cl in cdata: - s1 = string.split(cl,"=") - clist.append(s1[0].strip()) -clist = list(set(clist)) -clist.sort() - -for tl in tdata: - s1 = string.split(tl,"=") - tlist.append(s1[0].strip()) - set(tlist) -tlist = list(set(tlist)) -tlist.sort() - -# Use list comprehensions to print variables in clist but not tlist, and tlist but clist -conly = [x for x in clist if x not in tlist] -tonly = [x for x in tlist if x not in clist] -print("") -print(" TOTAL EXISTING CONF ENTRIES: %04d" % (int(len(clist)))) -print(" TOTAL NEW CONF ENTRIES : %04d" % (int(len(tlist)-len(clist)))) -print(" TOTAL AFTER UPDATE : %04d" % (int(len(tlist)))) -print("") -if conly == []: - print(" NO OBSOLETE ITEMS.") -else: - print(" THE FOLLOWING CONF ITEMS ARE OBSOLETE:") - print("") - for i in conly: - print(" + "+i) - print("") - print(" ********* PLEASE UPDATE LYX DOCUMENTATION! ************") -print("") -if tonly == []: - print(" NO NEW ITEMS.") -else: - print(" THE FOLLOWING CONF ITEMS ARE NEW:") - print("") - for i in tonly: - print(" + "+i) - print("") - print(" ********* PLEASE UPDATE LYX DOCUMENTATION! ************") - -# Write out data to files -cinspect = open("%s/current.conf" % (TMP_CONF_LIST_DIR),"w") -tinspect = open("%s/new.conf" % (TMP_CONF_LIST_DIR),"w") -for i in clist: - cinspect.write("%s\n" % (i)) -for i in tlist: - tinspect.write("%s\n" % (i)) -cinspect.close() -tinspect.close() -print("") -print(" INSPECT COMPARED FILES HERE:") -print(" %s/current.conf" % (TMP_CONF_LIST_DIR)) -print(" %s/new.conf" % (TMP_CONF_LIST_DIR)) - -# If the user requested update, do that here. -if mode=='update': - if conly!=[] or tonly!=[]: - print("") - print(" REPLACING FILE...") - cmd = 'cp %s/new.conf %s' % (TMP_CONF_LIST_DIR,STATIC_CONF_LIST) - call(cmd) - else: - print("") - print(" NO CHANGES NEEDED TO FILE %s" % (STATIC_CONF_LIST)) - -# Clean up -print("") -print(" CLEANING UP...") -if DEBUG: - cmd = 'rm -rf %s/unique.conf %s/tmp.conf' % (TMP_CONF_LIST_DIR,TMP_CONF_LIST_DIR) -else: - cmd = 'rm -rf %s/unique.conf %s/tmp.conf' % (TMP_CONF_LIST_DIR,TMP_CONF_LIST_DIR) -call(cmd) - -print("") -print(" DONE.") diff --git a/internal_tests/conf_list.static b/internal_tests/conf_list.static deleted file mode 100644 index ee80d0bf03..0000000000 --- a/internal_tests/conf_list.static +++ /dev/null @@ -1,450 +0,0 @@ -ADECK_FILE_PREFIX -ADECK_TRACK_DATA_DIR -AMODEL -ANLY_ASCII_REGEX_LEAD -ANLY_NC_TILE_REGEX -ANLY_TILE_PREFIX -ANLY_TILE_REGEX -BACKGROUND_MAP -BASIN -BDECK_FILE_PREFIX -BDECK_TRACK_DATA_DIR -BMODEL -CIRCLE_MARKER_SIZE -CI_METHOD -CONFIG_DIR -CONFIG_FILE -CONVERT_EXE -CROSS_MARKER_SIZE -CUT_EXE -CYCLONE -CYCLONE_INIT_DATE -CYCLONE_INIT_HR -CYCLONE_INPUT_DIR -CYCLONE_MODEL -CYCLONE_OUTPUT_DIR -CYCLONE_PLOT_TITLE -DEMO_YR -DEP_VARS -DLAND_FILE -DLAT -DLON -EGREP_EXE -ENSEMBLE_STAT_CONFIG -ENSEMBLE_STAT_CONFIG_FILE -ENSEMBLE_STAT_ENS_THRESH -ENSEMBLE_STAT_GRID_VX -ENSEMBLE_STAT_MET_OBS_ERROR_TABLE -ENSEMBLE_STAT_N_MEMBERS -ENSEMBLE_STAT_OUTPUT_DIR -ENS_VAR1_LEVELS -ENS_VAR1_NAME -ENS_VAR1_THRESH -EVENT_EQUALIZATION -EXTRACT_TILES_FILTER_OPTS -EXTRACT_TILES_OUTPUT_DIR -EXTRACT_TILES_VAR_LIST -FCST_ASCII_REGEX_LEAD -FCST_ENSEMBLE_STAT_INPUT_DIR -FCST_ENSEMBLE_STAT_INPUT_TEMPLATE -FCST_GRID_STAT_INPUT_DIR -FCST_GRID_STAT_INPUT_TEMPLATE -FCST_IS_DAILY_FILE -FCST_IS_PROB -FCST_MAX_FORECAST -FCST_MIN_FORECAST -FCST_MODE_INPUT_DATATYPE -FCST_MODE_INPUT_DIR -FCST_MODE_INPUT_TEMPLATE -FCST_MTD_CONV_RADIUS -FCST_MTD_CONV_THRESH -FCST_MTD_INPUT_DATATYPE -FCST_MTD_INPUT_DIR -FCST_MTD_INPUT_TEMPLATE -FCST_MXUPHL_5000-2000_THRESH -FCST_NC_TILE_REGEX -FCST_PCP_COMBINE_1_FIELD_NAME -FCST_PCP_COMBINE_6_FIELD_NAME -FCST_PCP_COMBINE_INPUT_DATATYPE -FCST_PCP_COMBINE_INPUT_DIR -FCST_PCP_COMBINE_INPUT_LEVEL -FCST_PCP_COMBINE_INPUT_TEMPLATE -FCST_PCP_COMBINE_IS_DAILY_FILE -FCST_PCP_COMBINE_MAX_FORECAST -FCST_PCP_COMBINE_METHOD -FCST_PCP_COMBINE_MIN_FORECAST -FCST_PCP_COMBINE_OUTPUT_DIR -FCST_PCP_COMBINE_OUTPUT_TEMPLATE -FCST_PCP_COMBINE_RUN -FCST_POINT_STAT_INPUT_DIR -FCST_POINT_STAT_INPUT_TEMPLATE -FCST_REFC_0_THRESH -FCST_TILE_PREFIX -FCST_TILE_REGEX -FCST_VAR10_LEVELS -FCST_VAR10_NAME -FCST_VAR10_OPTIONS -FCST_VAR11_LEVELS -FCST_VAR11_NAME -FCST_VAR11_OPTIONS -FCST_VAR12_LEVELS -FCST_VAR12_NAME -FCST_VAR12_OPTIONS -FCST_VAR13_LEVELS -FCST_VAR13_NAME -FCST_VAR13_OPTIONS -FCST_VAR14_LEVELS -FCST_VAR14_NAME -FCST_VAR14_OPTIONS -FCST_VAR15_LEVELS -FCST_VAR15_NAME -FCST_VAR15_OPTIONS -FCST_VAR16_LEVELS -FCST_VAR16_NAME -FCST_VAR16_OPTIONS -FCST_VAR17_LEVELS -FCST_VAR17_NAME -FCST_VAR17_OPTIONS -FCST_VAR18_LEVELS -FCST_VAR18_NAME -FCST_VAR18_OPTIONS -FCST_VAR1_LEVELS -FCST_VAR1_NAME -FCST_VAR1_OPTIONS -FCST_VAR1_THRESH -FCST_VAR2_LEVELS -FCST_VAR2_NAME -FCST_VAR2_OPTIONS -FCST_VAR2_THRESH -FCST_VAR3_LEVELS -FCST_VAR3_NAME -FCST_VAR3_OPTIONS -FCST_VAR3_THRESH -FCST_VAR4_LEVELS -FCST_VAR4_NAME -FCST_VAR4_OPTIONS -FCST_VAR4_THRESH -FCST_VAR5_LEVELS -FCST_VAR5_NAME -FCST_VAR5_OPTIONS -FCST_VAR5_THRESH -FCST_VAR6_LEVELS -FCST_VAR6_NAME -FCST_VAR6_OPTIONS -FCST_VAR7_LEVELS -FCST_VAR7_NAME -FCST_VAR7_OPTIONS -FCST_VAR8_LEVELS -FCST_VAR8_NAME -FCST_VAR8_OPTIONS -FCST_VAR9_LEVELS -FCST_VAR9_NAME -FCST_VAR9_OPTIONS -FILTER -FILTERED_TCST_DATA_FILE -FOOTNOTE_FLAG -FORECAST_TMPL -GEMPAKTOCF_INPUT_DIR -GEMPAKTOCF_INPUT_TEMPLATE -GEMPAKTOCF_OUTPUT_DIR -GEMPAKTOCF_OUTPUT_TEMPLATE -GENERATE_TRACK_ASCII -GEN_SEQ -GFS_ANLY_FILE_TMPL -GFS_FCST_FILE_TMPL -GRID_STAT_CONFIG -GRID_STAT_CONFIG_FILE -GRID_STAT_NEIGHBORHOOD_SHAPE -GRID_STAT_NEIGHBORHOOD_WIDTH -GRID_STAT_ONCE_PER_FIELD -GRID_STAT_OUTPUT_DIR -GRID_STAT_OUTPUT_TEMPLATE -GRID_STAT_VERIFICATION_MASK_TEMPLATE -HFIP_BASELINE -HREF_BASE -HRRRE_BASE -INIT_BEG -INIT_END -INIT_EXCLUDE -INIT_HOUR_BEG -INIT_HOUR_END -INIT_HOUR_INCREMENT -INIT_HOUR_METHOD -INIT_INCLUDE -INIT_INCREMENT -INIT_SEQ -INIT_TIME_FMT -INPUT_BASE -INTERP -JOB_ARGS -JOB_NAME -LAT_ADJ -LEAD -LEAD_LIST -LEAD_SEQ -LEAD_SEQ_1 -LEAD_SEQ_1_LABEL -LEAD_SEQ_2 -LEAD_SEQ_2_LABEL -LEAD_SEQ_3 -LEAD_SEQ_3_LABEL -LEAD_SEQ_4 -LEGEND -LINE_TYPE -LOG_DIR -LOG_LEVEL -LOG_METPLUS -LOG_MET_OUTPUT_TO_METPLUS -LOG_MET_VERBOSITY -LOG_TIMESTAMP_TEMPLATE -LOG_TIMESTAMP_USE_DATATIME -LON_ADJ -LOOP_BY -LOOP_ORDER -MASK_DIR -METPLUS_CONF -MET_BASE -MET_BIN -MET_INSTALL_DIR -MISSING_VAL -MISSING_VAL_TO_REPLACE -MODEL -MODEL1_NAME -MODEL1_NAME_ON_PLOT -MODEL1_OBS_NAME -MODEL1_STAT_DIR -MODEL_DATA_DIR -MODEL_INPUT_DIR -MODEL_MASK_TEMPLATE -MODEL_NAME -MODEL_PCP_OTEMPLATE -MODEL_TEMPLATE -MODE_CONFIG -MODE_CONV_RADIUS -MODE_CONV_THRESH -MODE_MERGE_FLAG -MODE_MERGE_THRESH -MODE_OUTPUT_DIR -MODE_QUILT -MODE_VERIFICATION_MASK_TEMPLATE -MTD_CONFIG -MTD_OUTPUT_DIR -MTD_OUTPUT_TEMPLATE -MTD_SINGLE_DATA_SRC -MTD_SINGLE_RUN -NCAP2_EXE -NCDUMP_EXE -NLAT -NLON -NO_EE -NO_LOG -OBSDIR -OBS_ENSEMBLE_STAT_POINT_INPUT_DIR -OBS_ENSEMBLE_STAT_POINT_INPUT_TEMPLATE -OBS_ENSEMBLE_STAT_WINDOW_BEGIN -OBS_ENSEMBLE_STAT_WINDOW_END -OBS_GRID_STAT_INPUT_DIR -OBS_GRID_STAT_INPUT_TEMPLATE -OBS_IS_DAILY_FILE -OBS_IS_PROB -OBS_LEVEL -OBS_MODE_INPUT_DATATYPE -OBS_MODE_INPUT_DIR -OBS_MODE_INPUT_TEMPLATE -OBS_MTD_CONV_RADIUS -OBS_MTD_CONV_THRESH -OBS_MTD_INPUT_DIR -OBS_MTD_INPUT_TEMPLATE -OBS_MXUPHL_500_THRESH -OBS_MergedReflectivityQCComposite_500_THRESH -OBS_NAME -OBS_PCP_COMBINE_12_FIELD_NAME -OBS_PCP_COMBINE_1_FIELD_NAME -OBS_PCP_COMBINE_24_FIELD_NAME -OBS_PCP_COMBINE_3_FIELD_NAME -OBS_PCP_COMBINE_6_FIELD_NAME -OBS_PCP_COMBINE_DATA_INTERVAL -OBS_PCP_COMBINE_INPUT_DATATYPE -OBS_PCP_COMBINE_INPUT_DIR -OBS_PCP_COMBINE_INPUT_LEVEL -OBS_PCP_COMBINE_INPUT_TEMPLATE -OBS_PCP_COMBINE_IS_DAILY_FILE -OBS_PCP_COMBINE_METHOD -OBS_PCP_COMBINE_OUTPUT_DIR -OBS_PCP_COMBINE_OUTPUT_TEMPLATE -OBS_PCP_COMBINE_RUN -OBS_PCP_COMBINE_TIMES_PER_FILE -OBS_POINT_STAT_INPUT_DIR -OBS_POINT_STAT_INPUT_TEMPLATE -OBS_REGRID_DATA_PLANE_INPUT_DIR -OBS_REGRID_DATA_PLANE_INPUT_TEMPLATE -OBS_REGRID_DATA_PLANE_OUTPUT_DIR -OBS_REGRID_DATA_PLANE_OUTPUT_TEMPLATE -OBS_REGRID_DATA_PLANE_RUN -OBS_REGRID_DATA_PLANE_TEMPLATE -OBS_VAR1_LEVELS -OBS_VAR1_NAME -OBS_VAR1_OPTIONS -OBS_VAR1_THRESH -OBS_VAR2_LEVELS -OBS_VAR2_NAME -OBS_VAR2_OPTIONS -OBS_VAR2_THRESH -OBS_VAR3_OPTIONS -OBS_VAR4_OPTIONS -OBS_VAR5_OPTIONS -OBS_WINDOW_BEGIN -OBS_WINDOW_END -OBTYPE -OUTPUT_BASE -OVERWRITE_TRACK -PARM_BASE -PB2NC_CONFIG_FILE -PB2NC_GRID -PB2NC_INPUT_DIR -PB2NC_INPUT_TEMPLATE -PB2NC_MESSAGE_TYPE -PB2NC_OBS_BUFR_VAR_LIST -PB2NC_OUTPUT_DIR -PB2NC_OUTPUT_TEMPLATE -PB2NC_POLY -PB2NC_SKIP_IF_OUTPUT_EXISTS -PB2NC_STATION_ID -PB2NC_TIME_SUMMARY_BEG -PB2NC_TIME_SUMMARY_END -PB2NC_TIME_SUMMARY_FLAG -PB2NC_TIME_SUMMARY_TYPES -PB2NC_TIME_SUMMARY_VAR_NAMES -PB2NC_VERTICAL_LOCATION -PB2NC_WINDOW_BEGIN -PB2NC_WINDOW_END -PLOTTING_OUTPUT_DIR -PLOTTING_SCRIPTS_DIR -PLOT_CONFIG_OPTS -PLOT_STATS_LIST -PLOT_TIME -PLOT_TYPES -POINT_STAT_CONFIG_FILE -POINT_STAT_GRID -POINT_STAT_MESSAGE_TYPE -POINT_STAT_OUTPUT_DIR -POINT_STAT_OUTPUT_TEMPLATE -POINT_STAT_POLY -POINT_STAT_REGRID_TO_GRID -POINT_STAT_STATION_ID -POINT_STAT_VERIFICATION_MASK_TEMPLATE -PREFIX -PROCESS_LIST -PROJ_DIR -REFERENCE_TMPL -REGION_LIST -REGRID_DATA_PLANE_METHOD -REGRID_DATA_PLANE_VERIF_GRID -REGRID_DATA_PLANE_WIDTH -REGRID_USING_MET_TOOL -RM_EXE -RP_DIFF -SAVE -SAVE_DATA -SCATTER_X -SCATTER_Y -SERIES -SERIES_ANALYSIS_BY_INIT_CONFIG_FILE -SERIES_ANALYSIS_BY_LEAD_CONFIG_FILE -SERIES_ANALYSIS_FILTER_OPTS -SERIES_BY_INIT_FILTERED_OUTPUT_DIR -SERIES_BY_INIT_OUTPUT_DIR -SERIES_BY_LEAD_FILTERED_OUTPUT_DIR -SERIES_BY_LEAD_GROUP_FCSTS -SERIES_BY_LEAD_OUTPUT_DIR -SERIES_CI -SKILL_REF -STAGING_DIR -STAT_ANALYSIS_CONFIG -STAT_ANALYSIS_DUMP_ROW_TMPL -STAT_ANALYSIS_LOOKIN_DIR -STAT_ANALYSIS_OUTPUT_DIR -STAT_ANALYSIS_OUT_STAT_TMPL -STAT_FILES_INPUT_DIR -STAT_LIST -STORM_ID -STORM_NAME -SUBTITLE -TCMPR_DATA_DIR -TCMPR_PLOT_OUTPUT_DIR -TC_PAIRS_CONFIG_FILE -TC_PAIRS_DIR -TC_PAIRS_FORCE_OVERWRITE -TC_STAT_AMODEL -TC_STAT_BASIN -TC_STAT_BMODEL -TC_STAT_CMD_LINE_JOB -TC_STAT_COLUMN_STR_NAME -TC_STAT_COLUMN_STR_VAL -TC_STAT_COLUMN_THRESH_NAME -TC_STAT_COLUMN_THRESH_VAL -TC_STAT_CONFIG_FILE -TC_STAT_CYCLONE -TC_STAT_DESC -TC_STAT_DIR -TC_STAT_INIT_BEG -TC_STAT_INIT_END -TC_STAT_INIT_EXCLUDE -TC_STAT_INIT_HOUR -TC_STAT_INIT_INCLUDE -TC_STAT_INIT_MASK -TC_STAT_INIT_STR_NAME -TC_STAT_INIT_STR_VAL -TC_STAT_INIT_THRESH_NAME -TC_STAT_INIT_THRESH_VAL -TC_STAT_INPUT_DIR -TC_STAT_JOBS_LIST -TC_STAT_LANDFALL -TC_STAT_LANDFALL_BEG -TC_STAT_LANDFALL_END -TC_STAT_LEAD -TC_STAT_LEAD_REQ -TC_STAT_MATCH_POINTS -TC_STAT_OUTPUT_DIR -TC_STAT_RUN_VIA -TC_STAT_STORM_ID -TC_STAT_STORM_NAME -TC_STAT_TRACK_WATCH_WARN -TC_STAT_VALID_BEG -TC_STAT_VALID_END -TC_STAT_VALID_EXCLUDE -TC_STAT_VALID_HOUR -TC_STAT_VALID_INCLUDE -TC_STAT_VALID_MASK -TC_STAT_WATER_ONLY -TIME_METHOD -TITLE -TMP_DIR -TOP_LEVEL_DIRS -TRACK_DATA_DIR -TRACK_DATA_MOD_FORCE_OVERWRITE -TRACK_DATA_SUBDIR_MOD -TRACK_TYPE -TR_EXE -VALID_BEG -VALID_END -VALID_HOUR_BEG -VALID_HOUR_END -VALID_HOUR_INCREMENT -VALID_HOUR_METHOD -VALID_INCREMENT -VALID_TIME_FMT -VAR1_FOURIER_DECOMP -VAR1_WAVE_NUM_LIST -VAR_LIST -VERIFICATION_GRID -VERIF_CASE -VERIF_GRID -VERIF_TYPE -WGRIB2 -XLAB -XLIM -YLAB -YLIM -img_res -{init?fmt diff --git a/internal_tests/deprecated/DeprecatedConfig b/internal_tests/deprecated/DeprecatedConfig deleted file mode 100755 index f94feabfb5..0000000000 --- a/internal_tests/deprecated/DeprecatedConfig +++ /dev/null @@ -1,225 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Grid-Stat configuration file. -// -// For additional information, see the MET_BASE/config/README file. -// -//////////////////////////////////////////////////////////////////////////////// - -// -// Output model name to be written -// -model = "${MODEL}"; - -// -// Output description to be written -// May be set separately in each "obs.field" entry -// -desc = "NA"; - -// -// Output observation type to be written -// -obtype = "${OBTYPE}"; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification grid -// -regrid = { - to_grid = ${GRID_VX}; - method = BILIN; - width = 2; - vld_thresh = 0.5; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// May be set separately in each "field" entry -// -censor_thresh = []; -censor_val = []; -cat_thresh = []; -cnt_thresh = [ NA ]; -cnt_logic = UNION; -wind_thresh = [ NA ]; -wind_logic = UNION; -eclv_points = 0.05; -nc_pairs_var_str = ""; -rank_corr_flag = FALSE; - -// -// Forecast and observation fields to be verified -// -fcst = { - field = [ ${FCST_FIELD} ]; - }; - -obs = { - field = [ ${OBS_FIELD} ]; - }; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Climatology data -// -climo_mean = fcst; -climo_mean = { - file_name = [ "${INPUT_BASE}/grid_to_grid/nwprod/fix/cmean_1d.1959${MET_VALID_HHMM}" ]; - - regrid = { - method = BILIN; - width = 2; - vld_thresh = 0.5; - shape = SQUARE; - } - - time_interp_method = NEAREST; - match_month = TRUE; - match_day = TRUE; - time_step = 21600; -} - -climo_stdev = climo_mean; -climo_stdev = { - file_name = []; -} - -climo_cdf_bins = 1; -write_cdf_bins = FALSE; - -//////////////////////////////////////////////////////////////////////////////// - -// -// Verification masking regions -// -mask = { - grid = [ "FULL" ]; - poly = [ "${INPUT_BASE}/grid_to_grid/poly/NHX.nc", - "${INPUT_BASE}/grid_to_grid/poly/SHX.nc", - "${INPUT_BASE}/grid_to_grid/poly/TRO.nc", - "${INPUT_BASE}/grid_to_grid/poly/PNA.nc" ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Confidence interval settings -// -ci_alpha = [ 0.05 ]; - -boot = { - interval = PCTILE; - rep_prop = 1.0; - n_rep = 0; - rng = "mt19937"; - seed = ""; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Data smoothing methods -// -interp = { - field = BOTH; - vld_thresh = 1.0; - shape = SQUARE; - - type = [ - { - method = NEAREST; - width = 1; - } - ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Neighborhood methods -// -nbrhd = { - field = BOTH; - width = [ 1 ]; - cov_thresh = [ >=0.5 ]; - vld_thresh = 1.0; - shape = SQUARE; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Fourier decomposition -// -fourier = { - wave_1d_beg = []; - wave_1d_end = []; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Gradient statistics -// May be set separately in each "obs.field" entry -// -gradient = { - dx = [ 1 ]; - dy = [ 1 ]; -} - -//////////////////////////////////////////////////////////////////////////////// - -// -// Statistical output types -// -output_flag = { - fho = NONE; - ctc = NONE; - cts = NONE; - mctc = NONE; - mcts = NONE; - cnt = NONE; - sl1l2 = NONE; - sal1l2 = STAT; - vl1l2 = NONE; - val1l2 = STAT; - vcnt = NONE; - pct = NONE; - pstd = NONE; - pjc = NONE; - prc = NONE; - eclv = NONE; - nbrctc = NONE; - nbrcts = NONE; - nbrcnt = NONE; - grad = NONE; -} - -// -// NetCDF matched pairs output file -// -nc_pairs_flag = { - latlon = FALSE; - raw = FALSE; - diff = FALSE; - climo = FALSE; - weight = FALSE; - nbrhd = FALSE; - fourier = FALSE; - gradient = FALSE; - apply_mask = FALSE; -} - -//////////////////////////////////////////////////////////////////////////////// - -grid_weight_flag = COS_LAT; -tmp_dir = "/tmp"; -output_prefix = "${FCST_VAR}_vs_${OBS_VAR}"; -//version = "V8.1"; - -//////////////////////////////////////////////////////////////////////////////// diff --git a/internal_tests/deprecated/deprecated.conf b/internal_tests/deprecated/deprecated.conf deleted file mode 100644 index 9eb9cf8d32..0000000000 --- a/internal_tests/deprecated/deprecated.conf +++ /dev/null @@ -1,16 +0,0 @@ -# METplus config used to test deprecated variable handling - -[config] -FCST_VAR1_NAME = TMP -FCST_VAR1_LEVELS = P850, P500, P250 - -GRID_STAT_CONFIG_FILE = {METPLUS_BASE}/internal_tests/deprecated/DeprecatedConfig - -OBS_PCP_COMBINE_INPUT_LEVEL = 6 - -STAT_LIST = TOTAL, OBAR, FBAR - -[dir] -MET_INSTALL_DIR = /location/of/met -INPUT_BASE = /home/{ENV[USER]}/deprecated-test-input -OUTPUT_BASE = /home/{ENV[USER]}/deprecated-test-output \ No newline at end of file diff --git a/internal_tests/environment_dev.yml b/internal_tests/environment_dev.yml deleted file mode 100644 index 6458c76980..0000000000 --- a/internal_tests/environment_dev.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: metplus_dev -channels: - - defaults -dependencies: - - python=3.6.3 - - sphinx-gallery - - rst2pdf - - pytest - - cartopy - - pandas - - sphinx_rtd_theme -prefix: /home/met_test/.conda/envs/metplus_dev - diff --git a/internal_tests/find_conf_var.py b/internal_tests/find_conf_var.py deleted file mode 100755 index 2b346b0c40..0000000000 --- a/internal_tests/find_conf_var.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/local/python/bin/python -# -# File: find_conf_var.py -# -# Author: D. Adriaansen -# -# Date: 12 Sept 2018 -# -# Purpose: Locate the use of a configuration variable within METplus source and conf areas -# -# Notes: You can search *just* conf areas via ./find_conf_var.py conf, or -# search *just* source areas via ./find_conf_var.py py, or -# search BOTH via ./find_conf_var.py -# -#________________________________________________________________________________________ - -# Python modules -import subprocess, sys, string - -# Path to METplus installation -mpinstall = '/home/dadriaan/projects/METplus' - -# Get user variables -cv = sys.argv[1] - -# What mode? -# 'conf' = only show conf files -# 'py' = only show py files -# 'all' = show all files -if len(sys.argv)==2: - mode = 'all' -elif len(sys.argv)==3: - mode = sys.argv[2] -else: - print("") - print("PLEASE PROVIDE CORRECT ARGUMENTS:") - print("") - print("./find_conf_var.py ") - print("-or-") - print("./find_conf_var.py ") - print("where is either py or conf") - print("") - sys.exit(1) - -# Split variable if appropriate -#if "_" in cv: -# s1 = string.split(cv,"_") -#else: -# s1 = [] -s1 = [] - -if s1 == []: - if mode=='conf': - cmd = 'grep -H %s %s/parm/metplus_config/*.conf %s/parm/use_cases/*/*.conf %s/parm/use_cases/*/*/*.conf %s/ush/*.py | grep -v \'\.py\''% (cv,mpinstall,mpinstall,mpinstall,mpinstall) - elif mode=='py': - cmd = 'grep -H %s %s/parm/metplus_config/*.conf %s/parm/use_cases/*/*.conf %s/parm/use_cases/*/*/*.conf %s/ush/*.py | grep -v \'\.conf\''% (cv,mpinstall,mpinstall,mpinstall,mpinstall) - else: - cmd = 'grep -H %s %s/parm/metplus_config/*.conf %s/parm/use_cases/*/*.conf %s/parm/use_cases/*/*/*.conf %s/ush/*.py'% (cv,mpinstall,mpinstall,mpinstall,mpinstall) - #print(cmd) - subprocess.call('%s' % (cmd), shell=True, executable='/bin/csh') -else: - for i in s1: - cmd = 'grep -H _%s %s/parm/metplus_config/*.conf %s/parm/use_cases/*/*.conf %s/parm/use_cases/*/*/*.conf %s/ush/*.py'% (i,mpinstall,mpinstall,mpinstall,mpinstall) - #print(cmd) - subprocess.call('%s' % (cmd), shell=True, executable='/bin/csh') diff --git a/internal_tests/produtil/example-produtil-run.py b/internal_tests/produtil/example-produtil-run.py deleted file mode 100755 index 5a5aad24a3..0000000000 --- a/internal_tests/produtil/example-produtil-run.py +++ /dev/null @@ -1,85 +0,0 @@ -#! /usr/bin/env python - -import logging, os -import produtil.setup -import produtil.run -from produtil.run import run, checkrun, exe, mpirun, mpi, alias, \ - ExitStatusException, runstr - -# Initialize produtil. The arguments I send are not necessary, but -# they do make the output more elegant. -produtil.setup.setup( - send_dbn=False, # eliminate two "I can't find dbn_alert" warnings - jobname="example") # Set the name of the job - -logger=logging.getLogger('my/logging/domain') - -logger.info('hello world examples') -logger.info('single argument to echo') -run(exe("echo")["hello world"]) # prints => hello world - -logger.info('two arguments to echo') -run(exe("echo")["hello","world"]) # prints => hello world - -logger.info('generate command step-by-step') -cmd=exe("echo") -for s in [ "hello", "world" ]: - cmd=cmd[s] -run(cmd) # prints => hello world - -logger.info('pass environment variables to sh') -cmd=exe("sh")['-c','$COMMAND $ENVAR1 $ENVAR2'].env( - COMMAND='echo', ENVAR1='hello', ENVAR2='world') -run(cmd) # runs sh to print => hello world - -logger.info('send a string to cat') -mytext='hello world\n' -run(exe('cat') << mytext) # prints => hello world - -logger.info('send a file to cat') -with open('hello_world.txt','wt') as f: - f.write('hello world\n') -run(exe('cat') < 'hello_world.txt') - -logger.info('capture a string and print it') -hello_world = runstr(exe('echo')['hello world']) -logger.info('printed => %s'%(hello_world.strip(),)) - -logger.info( 'true/false exit status examples') - -# "checkrun" example. This will raise an exception if the program fails -for progname in [ '/bin/true', '/bin/false' ]: - try: - checkrun(exe(progname)) - logger.info('%s: Success! Rejoice! Exit status 0!'%(progname,)) - except ExitStatusException as ese: - returncode=ese.returncode - logger.error('OHNO! OHNO! %s returncode is %d'%(progname,returncode)) - -logger.info('logging examples with serial programs') - -logger=logging.getLogger('example/domain') -run(exe('echo')['hello world'],logger=logger) -# prints => hello world -# but logs the execution of the program and the exit status - -## Now we add logging. Let's run a program that exits with status 3 -run(exe("sh")['-c','echo will exit 3 ; exit 3'],logger=logger) - -logger.info('mpi examples. NOTE: this will fail unless you are in a batch job') - -## MPI example, without logging. I'm going to pretend "echo" is an -## MPI program. If you had a real MPI program, then you would replace -## the initial cmd= line. - -cmd=mpi('echo')['hello world'] * 24 # 24 ranks of echo hello world -cmd=mpirun(cmd) # convert to local machine's MPI execution syntax -print 'will run '+cmd.to_shell() # just for fun, print the command it will execute -run(cmd) - -# MPMD example: twelve ranks each of "echo hello world" and -# "echo goodbye world". This time, we'll use the logging -# functionality so produtil will print what it is going to do. -cmd=mpi('echo')['hello world'] * 12 + mpi('echo')['goodbye world'] * 12 -cmd=mpirun(cmd,logger=logger) -run(cmd,logger=logger) diff --git a/internal_tests/produtil/exampletest.py b/internal_tests/produtil/exampletest.py deleted file mode 100755 index 11beae14ba..0000000000 --- a/internal_tests/produtil/exampletest.py +++ /dev/null @@ -1,128 +0,0 @@ -#! /usr/bin/env python - -from produtil.testing.tokenize import Tokenizer -from produtil.testing.parse import Parser -from produtil.testing.parsetree import fileless_context, Scope - -mystring = ''' - -# Example of platform auto-detection. - -# Note to self: need to implement an "embed python" directive. - -platform wcoss.cray { - NWPROD="/gpfs/hps/nco/ops/nwprod/" - embed bash detect [[[ - # This function is used at PARSE TIME to detect whether we are - # on WCOSS Cray. It must be very fast and low resource usage - # since the parser runs it. - if [[ -d /gpfs/hps && -e /etc/SuSE-release && -d /usrx ]] ; then - exit 0 - fi - exit 1 - ]]] -} - -platform wcoss.phase1 { - NWPROD="/gpfs/hps/nco/ops/nwprod/" - embed bash detect [[[ - # This function is used at PARSE TIME to detect whether we are - # on WCOSS Phase 1. It must be very fast and low resource - # usage since the parser runs it. - if [[ -d /usrx && -d /global && -e /etc/redhat-release && \ - -e /etc/prod ]] ; then - # We are on WCOSS Phase 1 or 2. - if ( ! cat /proc/cpuinfo |grep 'processor.*32' ) ; then - # Fewer than 32 fake (hyperthreading) cpus, so Phase 1. - exit 0 - fi - fi - exit 1 - ]]] -} - -platform wcoss.phase2 { - NWPROD="/gpfs/hps/nco/ops/nwprod/" - embed bash detect [[[ - # This function is used at PARSE TIME to detect whether we are - # on WCOSS Phase 2. It must be very fast and low resource - # usage since the parser runs it. - if [[ -d /usrx && -d /global && -e /etc/redhat-release && \ - -e /etc/prod ]] ; then - # We are on WCOSS Phase 1 or 2. - if ( cat /proc/cpuinfo |grep 'processor.*32' ) ; then - # At least 32 fake (hyperthreading) cpus, so Phase 2. - exit 0 - fi - fi - exit 1 - ]]] -} - -platform theia { - NWPROD='/scratch3/NCEPDEV/nwprod' - embed bash detect [[[ - # This function is used at PARSE TIME to detect whether we are - # on NOAA Theia. It must be very fast and low resource usage - # since the parser runs it. - if [[ -d /scratch3 && -d /scratch4 && -d /contrib ]] ; then - exit 0 - fi - exit 1 - ]]] -} - -# Specify list of platforms and request autodetection. Resulting -# platform will be in the "plat" variable: - -autodetect plat (/ wcoss.phase1, wcoss.phase2, wcoss.cray, theia /) - - - -# Example variable hashes without "use" statements: - -gfspaths = { - HOMEgsm="@[NWPROD]/gfs.@[GFS_VERSION]" - PARMGLOBAL="@[HOMEgsm]/parm" - FIXGLOBAL="@[HOMEgsm]/fix" - EXECGLOBAL="@[HOMEgsm]/exec" -} - -enable_ndslfv = { - NDSLFV='.true.' - MASS_DP='.true.' - PROCESS_SPLIT='.false.' - dp_import=1 -} - - -# example with "use" statements: - -gfsvars={ - use gfspaths - use plat - use enable_ndslfv - nam_dyn=[[[ -&nam_dyn -# there would be many more variables here ... - ndslfv=@[NDSLFV], - mass_dp=@[MASS_DP], - process_split=@[PROCESS_SPLIT], -/ -]]] - GFS_VERSION="v13.0.4" -} -''' - -tokenizer=Tokenizer() -parser=Parser() -context=fileless_context() -scope=Scope() -parser.parse(tokenizer.tokenize(mystring,filename="(mystring)"),scope) - -def get(var): - return scope.resolve(var).string_context(context) - -print "nam_dyn = \n%s\n"%(get('gfsvars%nam_dyn'),) -print "NWPROD = %s\n"%(get('plat%NWPROD'),) -print "FIXGLOBAL = %s\n"%(get('gfsvars%FIXGLOBAL'),) diff --git a/internal_tests/produtil/forcetest.py b/internal_tests/produtil/forcetest.py deleted file mode 100644 index ada09681da..0000000000 --- a/internal_tests/produtil/forcetest.py +++ /dev/null @@ -1,88 +0,0 @@ -#! /usr/bin/env python - -import produtil.setup -from produtil.run import * - -produtil.setup.setup(send_dbn=False) - -impi=make_mpi('impi',total_tasks=48,force=True) -mpiexec=make_mpi('mpiexec',total_tasks=48,nodesize=24,force=True,silent=True) -lsf_cray_intel=make_mpi( - 'lsf_cray_intel',total_tasks=48,nodesize=24,aprun_path='aprun', - hyperthreads=1,p_state_turbo=2601000,force=True,silent=True) -inside_aprun=make_mpi('inside_aprun',force=True) -mpiexec_mpt=make_mpi('mpiexec_mpt',total_tasks=24,force=True,silent=True) -mpirun_lsf=make_mpi('mpirun_lsf',force=True,silent=True) -srun=make_mpi('srun',force=True,silent=True) -no_mpi=make_mpi(None,force=True) - -def mpiserial_ls_10(mpiimpl): - world='ls -l * 10 in mpiserial' - cmd=mpiserial(exe('ls',mpiimpl=mpiimpl)['-l']) - cmd=cmd*10 - cmd=mpirun(cmd,mpiimpl) - return world,cmd - -def world_n140_n50t2(mpiimpl): - world=mpi('prog1')*140+openmp(mpi('prog2')*50,threads=2,mpiimpl=mpiimpl) - mpirunner=mpirun(world,mpiimpl=mpiimpl,label_io=True) - return world,mpirunner - -def world_n1000(mpiimpl): - world=mpi('prog1')*1000 - mpirunner=mpirun(world,mpiimpl=mpiimpl,label_io=True) - return world,mpirunner - -def world_loop_n1000(mpiimpl): - world=mpi('prog1') - for i in xrange(999): - world+=mpi('prog1') - mpirunner=mpirun(world,mpiimpl=mpiimpl,label_io=True) - return world,mpirunner - -def allranks(mpiimpl): - world=mpi('prog1') - mpirunner=mpirun(world,mpiimpl=mpiimpl,label_io=True,allranks=True) - return world,mpirunner - -def runsync(mpiimpl): - world='runsync' - mpiimpl.runsync() - return world,'SUCCESS' - -def exe_hello_world(mpiimpl): - cmd=exe('/bin/echo',mpiimpl=mpiimpl)['hello','world'] - return '/bin/echo hello world - serial',cmd - -def omp_hello_world(mpiimpl): - cmd1=exe('/bin/echo',mpiimpl=mpiimpl)['hello','world'] - cmd2=openmp(cmd1,mpiimpl=mpiimpl,threads=24) - return '/bin/echo hello world - 24 threads',cmd2 - -for worlder in [ exe_hello_world, allranks, world_n140_n50t2, world_n1000, world_loop_n1000 ]: - first=True - print '------------------------------------------------------------------------' - for mpiimpl in [ lsf_cray_intel, impi, mpiexec, mpiexec_mpt, \ - mpirun_lsf, srun, inside_aprun, no_mpi ]: - try: - (world,mpirunner) = worlder(mpiimpl) - if first: - print 'COMMAND: %s'%(repr(world),) - print - first=False - if isinstance(mpirunner,str): - cmd=mpirunner - else: - cmd=mpirunner.to_shell() - if cmd.find('\n')<0: - print '%20s ==> %s'%(mpiimpl.name(),cmd) - else: - pre=mpiimpl.name() - arrow='==>' - for line in cmd.splitlines(): - print '%20s %s %s'%(pre,arrow,line) - pre=' ' - arrow=' ' - except Exception as e: - print '%20s (E) %s'%( - mpiimpl.name(),str(e)) diff --git a/internal_tests/produtil/testgen.py b/internal_tests/produtil/testgen.py deleted file mode 100755 index b61806f4ed..0000000000 --- a/internal_tests/produtil/testgen.py +++ /dev/null @@ -1,138 +0,0 @@ -#! /usr/bin/env python - -# FIXME: Remove if randstring is removed: -# import string, random - -import logging -import os -import sys -import getopt -import datetime - -import produtil.setup -produtil.setup.setup(send_dbn=False) - -from produtil.testing.testgen import TestGen -from produtil.testing.utilities import BASELINE, EXECUTION -from produtil.testing.tokenize import Tokenizer, TokenizeFile -from produtil.testing.parse import Parser -from produtil.testing.rocoto import RocotoRunner -from produtil.testing.script import BashRunner - -testgen=None - -def usage(why=None): - sys.stderr.write('''Generates a workflow to run a test suite. - - testgen [options] (-R | -B) (-b | -v) [--] /input/path /output/path - -Mandatory arguments: --R | -B - Mandatory output type: [-R]ocoto workflow or flat [-B]ash script. --b | -v - Run mode: generate a new [-b]aseline or [-v]erify results by - comparing to an old baseline. -/input/path - Path to input file that describes the tests. -/output/path - Where to create scripts; should be a parallel filesystem accessible - by compute nodes. Must not exist yet. - -Optional arguments: --d - Requests a dry run; print what would be done without doing it. --u integer - Specifies the unique positive integer id of this test run. - Default is to automatically generate one. --P - Enable Python profiling via cProfile. -''') - if why is not None: - sys.stderr.write('ABORTING: %s\n'%(why,)) - exit(2) - -# FIXME: Remove this if unneeded: -# def randstring(): -# """!Generates a random number from 0 to 62**6-1 and expresses it -# in base 62 as a six character string.""" -# sixtytwo = string.digits + string.lowercase + string.uppercase -# i1=int(random.uniform(0,62**3)) -# i2=int(random.uniform(0,62**3)) -# randstring = \ -# ''.join([ sixtytwo[min(61,max(0,(i1/62**j) % 62))] -# for j in xrange(3) ] + \ -# [ sixtytwo[min(61,max(0,(i2/62**j) % 62))] -# for j in xrange(3) ]) - -def parse_arguments(args): - run_mode=None - output_type=None - output_path=None - dry_run=False - unique_id=None - profile=False - - if len(args)<1: usage() - - optval,arglist=getopt.getopt(sys.argv[1:],'bvRBdu:P') - - for opt,val in optval: - if opt=='-R': - if output_type: - usage('specified output type (-R, -B) more than once') - output_type=RocotoRunner - elif opt=='-B': - if output_type: - usage('specified output type (-R, -B) more than once') - output_type=BashRunner - elif opt=='-b': - if run_mode: - usage('specified run mode (-b, -v) more than once') - run_mode=BASELINE - elif opt=='-v': - if run_mode: - usage('specified run mode (-b, -v) more than once') - run_mode=EXECUTION - elif opt=='-P': - profile=True - elif opt=='-u': - unique_id=int(val,10) - elif opt=='-d': - dry_run=True - else: - usage('unknown option '+opt) - - if not run_mode: - usage('run mode (-b or -v) unspecified') - elif not output_type: - usage('output type (-R or -B) unspecified') - elif len(arglist)!=2: - usage('exactly two non-option arguments were expected, not %d'%( - len(arglist),)) - - input_location=arglist[0] - output_path=arglist[1] - - if not os.path.exists(input_location): - usage('%s: does not exist'%(input_location,)) - - absinput=os.path.normpath(os.path.abspath(input_location)) - - return run_mode, output_type, output_path, absinput, dry_run, \ - unique_id, profile - -def main(args): - run_mode, OutputType, outloc, inloc, dry_run, unique_id, profile = \ - parse_arguments(args) - - global testgen # global scope for cProfile - testgen=TestGen(run_mode, OutputType, outloc, inloc, dry_run, unique_id) - - if profile: - import cProfile - cProfile.run('testgen.testgen()') - else: - testgen.testgen() - -if __name__=='__main__': - main(sys.argv[1:]) diff --git a/internal_tests/produtil/testme.py b/internal_tests/produtil/testme.py deleted file mode 100644 index 2b662828d1..0000000000 --- a/internal_tests/produtil/testme.py +++ /dev/null @@ -1,2645 +0,0 @@ -#! /usr/bin/env python - -import sys, re, StringIO, collections, os, datetime, logging -import produtil.run, produtil.log, produtil.setup - -##@var BASELINE -# A constant that indicates the suite is being run to generate a new baseline. -BASELINE=object() - -##@var EXECUTION -# A constant that indicates the suite is being run to verify against an existing baseline. -EXECUTION=object() - -run_mode=EXECUTION - -module_logger=logging.getLogger('produtil.testing') - -##@var bash_functions -# Functions used by the bash language scripts this package generates. -bash_functions=r''' -function deliver_file() { - local src tgt - set -e - src="$1" - tgt="$2" - if [[ -d "$tgt" ]] ; then - tgt="$tgt"/$( basename "$src" ) - fi - tmpfile="$tgt.$$.$RANDOM.$RANDOM" - cp -fp "$src" "$tmpfile" - mv -fT "$tmpfile" "$tgt" -} - -function bitcmp() { - local src tgt bn result - set -e - src="$1" - tgt="$2" - if [[ -d "$tgt" ]] ; then - bn=$( basename "$src" ) - tgt="$tgt/$bn" - elif [[ -d "$src" ]] ; then - bn=$( basename "$tgt" ) - src="$src/$bn" - fi - set +e - cmp "$src" "$tgt" - result=$? - set -e - return $result -} - -function atparse { - set +x # There is too much output if "set -x" is on. - set +u # We expect empty variables in this function. - set +e # We expect some evals to fail too. - # Use __ in names to avoid clashing with variables in {var} blocks. - local __text __before __after __during - for __text in "$@" ; do - if [[ $__text =~ ^([a-zA-Z][a-zA-Z0-9_]*)=(.*)$ ]] ; then - eval "local ${BASH_REMATCH[1]}" - eval "${BASH_REMATCH[1]}="'"${BASH_REMATCH[2]}"' - else - echo "ERROR: Ignoring invalid argument $__text\n" 1>&2 - fi - done - while IFS= read -r __text ; do - while [[ "$__text" =~ ^([^@]*)(@\[[a-zA-Z_][a-zA-Z_0-9]*\]|@\[\'[^\']*\'\]|@\[@\]|@)(.*) ]] ; do - __before="${BASH_REMATCH[1]}" - __during="${BASH_REMATCH[2]}" - __after="${BASH_REMATCH[3]}" -# printf 'PARSE[%s|%s|%s]\n' "$__before" "$__during" "$__after" - printf %s "$__before" - if [[ "$__during" =~ ^@\[\'(.*)\'\]$ ]] ; then - printf %s "${BASH_REMATCH[1]}" - elif [[ "$__during" == '@[@]' ]] ; then - printf @ - elif [[ "$__during" =~ ^@\[([a-zA-Z_][a-zA-Z_0-9]*)\] ]] ; then - set -u - eval 'printf %s "$'"${BASH_REMATCH[1]}"'"' - set +u - else - printf '%s' "$__during" - fi - if [[ "$__after" == "$__text" ]] ; then - break - fi - __text="$__after" - done - printf '%s\n' "$__text" - done -} -''' - -######################################################################## - -def elipses(long_string,max_length=20,elipses='...'): - strlen=len(long_string) - if strlen.) - | (?P [\]\[]+ ) - | (?P [^\\@\]\[]+) - | (?P - @ \[ @ \] - | @ \[ ' [^']+ ' \] - | @ \[ [^\]]+ \] - ) - | (?P @ (?!\[) ) - | (?P . ) - ) ''',dq): - if m.group('backslashed'): - s=m.group('backslashed') - if s=='@': - output.write('@[@]') - elif s in '[]': - output.write("@['"+s+"']") - else: - output.write(s) - elif m.group('literal_at'): - output.write('@[@]') - elif m.group('atblock'): - output.write(m.group('atblock')) - elif m.group('braces'): - output.write("@['"+m.group('braces')+"']") - elif m.group('text'): - output.write(m.group('text')) - else: - raise ValueError('Cannot convert double-quote string \"%s\" to brace string: parser error around character \"%s\"."'%(dq,m.group())) - value=output.getvalue() - output.close() - return value - -######################################################################## - -class Context: - def __init__(self,scopes,token,run_mode,logger): - self.run_mode=run_mode - self.token=token - self.scopes=scopes - if logger is None: - logger=module_logger - self.logger=logger - @property - def filename(self): - return self.token.filename - @property - def lineno(self): - return self.token.lineno - def info(self,message): - self.logger.info("%s: %s: %s"%( - str(self.token.filename), - repr(self.token.lineno), - message)) - def warning(self,message): - self.logger.warning("%s: %s: %s"%( - str(self.token.filename), - repr(self.token.lineno), - message)) - def error(self,message): - self.logger.error("%s: %s: %s"%( - str(self.token.filename), - repr(self.token.lineno), - message)) - -######################################################################## - -def is_valid_workflow_name(name): - return bool(re.match('(?s)^[a-zA-Z][a-zA-Z0-9_]*$',name)) - -def to_rocoto_walltime(seconds): - return '%02d:%02d:%02d'%( - seconds//3600, (seconds//60)%60, (seconds//1)%60) - -def as_xml_attr(val): - return val.replace('&','&').replace('<','<') \ - .replace('>','>').replace('"','"') \ - .replace("'",''').replace('\n',' ') - -######################################################################## - -class RocotoTask(object): - def __init__(self,name,obj): - super(RocotoTask,self).__init__() - self.__name=name - self.__obj=obj - def get_test_resources(self,con): - # Get resource information for everything except cpus and time: - try: - test_size=self.__obj.resolve('test_size') - except KeyError: - test_size='short' - if test_size not in [ 'short', 'long' ]: - raise ValueError('Test %s: test size %s is not "small" or "long"'%( - obj.name,repr(test_size))) - elif test_size=='long': - return self.__obj.defscopes[-1].resolve( - 'plat%rocoto%long_test_resources').string_context(con) - else: - return self.__obj.defscopes[-1].resolve( - 'plat%rocoto%short_test_resources').string_context(con) - def get_walltime(self,con): - # Get walltime requirements: - try: - return '%s'%( - to_rocoto_walltime(self.__obj.resolve('walltime') - .numeric_context(con)),) - except KeyError as ke: - return '%s'%( - to_rocoto_walltime(self.__obj.defscopes[-1] - .resolve('plat%DEFAULT_TEST_WALLTIME') - .numeric_context(con)),) - def get_cpu_resources(self,con): - execute=self.__obj.resolve('execute') - if isinstance(execute,SpawnProcess): - return execute.rocoto_resources(con) - else: - return '2' - def j_job_name(self,workflow,con): - return os.path.join(workflow.install_dir(con),'jobs', - 'J%s_%s'%(workflow.NAME,self.__name.upper())) - def ex_script_name(self,workflow,con): - return os.path.join(workflow.install_dir(con),'scripts', - 'ex%s_%s'%(workflow.name,self.__name.lower())) - def j_job_contents(self,workflow,con): - return r'''#! /usr/bin/env bash - -# DO NOT EDIT THIS SCRIPT; IT IS AUTOMATICALLY GENERATED - -{script} - -export HOME{workflow}="${{HOME{workflow}:-$RT_INSTALL_DIR}}" -export USH{workflow}="${{USH{workflow}:-$HOME{workflow}/ush}}" -export EX{workflow}="${{USH{workflow}:-$HOME{workflow}/scripts}}" - -{ex_script} -'''.format(script=self.__obj.resolve('prep').bash_context(con), - workflow=workflow.name,WORKFLOW=workflow.NAME, - ex_script=self.ex_script_name(workflow,con)) - def ex_script_contents(self,workflow,con): - return r'''#! /usr/bin/env bash - -# DO NOT EDIT THIS SCRIPT; IT IS AUTOMATICALLY GENERATED - -source $HOME{workflow}/ush/functions.bash - -set -xe - -{SCRIPT} -'''.format(SCRIPT=self.__obj.bash_context(con), - WORKFLOW=workflow.NAME,workflow=workflow.name) - def generate_xml(self,out,workflow,con): - out.write(r''' - set -xue ; cd "&INSTALL_DIR;/jobs" ; ./J{WORKFLOW}_{NAME} - rt_{name} - &ACCOUNT; - {test_resources} - {cpu_resources} - {walltime} - - {WORKFLOW}_INSTALL_DIR - &INSTALL_DIR; - - &LOG_DIR;/{name}.log -'''.format(name=self.__name,NAME=self.__name.upper(), - test_resources=self.get_test_resources(con), - cpu_resources=self.get_cpu_resources(con), - walltime=self.get_walltime(con), - WORKFLOW=workflow.name.upper())) - deps=[ dep for dep in self.__obj.iterdeps()] - if len(deps)==1: - out.write(' \n') - elif len(deps)>1: - out.write(' \n') - for dep in deps: - if isinstance(dep,Build): - out.write(' \n'%(dep.name,)) - elif isinstance(dep,Test): - out.write(' \n'%(dep.name,)) - if len(deps)==1: - out.write(' \n') - elif len(deps)>1: - out.write(' \n') - out.write(r''' - -''') - -######################################################################## - -class RocotoWorkflow(object): - def __init__(self,name,scope): - super(RocotoWorkflow,self).__init__() - if not is_valid_workflow_name(name): - raise Exception( - '%s: not a valid workflow name; must begin with a letter ' - 'and only contain alphanumerics and underscore. Fixme: ' - 'need better exception here.'%(name,)) - self.name=name.lower() - self.NAME=name.upper() - self.cycle=datetime.datetime.now().strftime('%Y%m%d%H%M') - self.__scope=scope - self.__tasklist=list() - self.__taskdict=dict() - self.__buildlist=list() - self.__namemap=dict() - self.__files=dict() - self.__install=None - self.__uninstall=None - self.__end_build=r'''else - echo ERROR: Invalid build "$1" 1>&2 - exit 1 -fi -''' - def generate_install_script(self): - assert(self.__install is not None) - if self.__install is None: - return '' - else: - return self.__install.getvalue()+self.__end_build - def generate_uninstall_script(self): - assert(self.__uninstall is not None) - if self.__uninstall is None: - return '' - else: - return self.__uninstall.getvalue()+self.__end_build - def has_builds(self): - return bool(self.__buildlist) - def getvar(self,var): - return self.__scope.resolve(var) - def as_attr(self,var,con): - val=self.getvar(var) - if isinstance(val,BaseObject): - val=val.string_context(con) - return as_xml_attr(val) - def run(self,obj,con): - if isinstance(obj,Test): - self.run_test(obj,con) - elif isinstance(obj,Build): - self.run_build(obj,con) - else: - raise Exception( - 'Rocoto can only run Test and Build objects, not a %s %s.'%( - type(obj).__name__,repr(obj))) - def run_test(self,test,con): - self.add_test(test.name,test) - # FIXME: INSERT CODE HERE - def run_build(self,build,con): - self.add_build(build.name) - - kwargs={ 'install_if':'elif', - 'uninstall_if':'elif', - 'name':build.name, - 'script':build.bash_context(con), - 'target':build.resolve('target').bash_context(con) - } - - if not self.__install: - self.__install=StringIO.StringIO() - self.__install.write(r'''#! /usr/bin/env bash - -# DO NOT EDIT THIS SCRIPT; IT IS AUTOMATICALLY GENERATED -# This script installs executables in the build_* jobs. - -set -xue - -''') - kwargs['install_if']='if' - - if not self.__uninstall: - self.__uninstall=StringIO.StringIO() - self.__uninstall.write(r'''#! /usr/bin/env bash - -# DO NOT EDIT THIS SCRIPT; IT IS AUTOMATICALLY GENERATED -# This script uninstalls executables installed by the "install.sh" script. - -set -xue - -''') - kwargs['uninstall_if']='if' - - - self.__install.write(r''' -{install_if} [[ "$1" == {name} ]] ; then -######################################################################## -###### BUILD SCRIPT FOR TARGET {name} -######################################################################## -rm -f {target} -{script} -set -xe -test -s {target} -test -x {target} -'''.format(**kwargs)) - self.__uninstall.write(r''' -{uninstall_if} [[ "$1" == {name} ]] ; then -rm -f {target} -'''.format(**kwargs)) - - - def add_file(self,path,contents): - path=os.path.normpath(path) - if path in self.__files: - raise Exception('%s: tried to redefine contents for this file. ' - 'Fixme: need better exception here.'%(path,)) - contents=str(contents) - self.__files[path]=contents - def add_build(self,buildname): - if buildname in self.__buildlist: - raise Exception('%s: duplicate build name. Fixme: need better ' - 'exception here.'%(buildname,)) - self.__buildlist.append(buildname) - def add_test(self,taskname,obj): - assert(taskname[0:5]!='test_') - taskname='test_'+str(taskname) - if taskname in self.__taskdict: - raise Exception('%s: tried to redefine task. Fixme: need better ' - 'exception here.'%(taskname,)) - task=RocotoTask(taskname,obj) - self.__tasklist.append(taskname) - self.__taskdict[taskname]=task - return task - def iter_buildnames(self): - for buildname in self.__buildlist: - yield buildname - def iter_paths(self): - for path in self.__files.iterkeys(): - yield path - def iter_testnames(self): - for taskname in self.__tasklist: - yield taskname - def iter_files(self): - for path,contents in self.__files.iteritems(): - yield path,contents - def iter_tests(self): - for taskname in self.__tasklist: - yield taskname,self.__taskdict[taskname] - def as_walltime(self,time,con): - time=to_rocoto_walltime(self.getvar(time).numeric_context(con)) - assert(time is not None) - return time - def install_dir(self,con): - idir=self.getvar('plat%rocoto%install_dir').string_context(con) - assert(idir.find('@')<0) - return idir - def generate_xml(self,out,con): - kwargs={ - 'account':self.as_attr('plat%CPU_ACCOUNT',con), - 'log_dir':self.as_attr('plat%rocoto%log_dir',con), - 'install_dir':self.as_attr('plat%rocoto%install_dir',con), - 'UNIQUE_ID':self.as_attr('UNIQUE_ID',con), - 'scheduler':self.as_attr('plat%rocoto%scheduler',con), - 'taskthrottle':5, - 'name':self.name, - 'cycle':'202012311830', - 'build_resources':self.getvar('plat%rocoto%build_resources').string_context(con), - 'build_walltime':self.as_walltime('plat%BUILD_WALLTIME',con) - } - out.write(r''' - - - - - - - -]> - - - - - - - {cycle:s} {cycle:s} 01:00:00 - - - &LOG_DIR;/rocoto_@Y@m@d@H.log - - -'''.format(**kwargs)) - - for testname,test in self.iter_tests(): - test.generate_xml(out,self,con) - - out.write(r''' - - -''') - - if self.has_builds(): - out.write(r''' - - '''.format(**kwargs)) - out.write(' '.join([ b for b in self.iter_buildnames()])) - out.write(r''' - - set -xue ; cd "&INSTALL_DIR;/src" ; ./install.sh "#BUILD#" - rt_build_#BUILD# - &ACCOUNT; - {build_walltime} - {build_resources:s} - - RT_INSTALL_DIR - &INSTALL_DIR; - - &LOG_DIR;/build_#BUILD#.log - - set -xue ; cd "&INSTALL_DIR;/src" ; ./uninstall.sh "#BUILD#" - - - - -'''.format(**kwargs)) - - out.write(r''' - - -''') - -######################################################################## - -class RocotoRunner(object): - def __init__(self): - super(RocotoRunner,self).__init__() - def make_runner(self,parser): - work=None - con=None - for runme,runme_context in parser.iterrun(): - if work is None: - work=RocotoWorkflow('rt',runme.defscopes[-1]) - work.run(runme,runme_context) - assert(runme_context is not None) - if con is None: - con=runme_context - assert(con is not None) - - if work is None: - raise ValueError('ERROR: No "run" statments seen; ' - 'nothing to do.\n'); - - assert(con is not None) - - def here(path): - here=os.path.normpath(os.path.join(work.install_dir(con),path)) - dir=os.path.normpath(os.path.dirname(here)) - if dir!=here: - produtil.fileop.makedirs(dir) - print here - return here - - with open(here('rocoto/workflow.xml'),'wt') as f: - work.generate_xml(f,con) - with open(here('src/install.sh'),'wt') as f: - f.write(work.generate_install_script()) - os.fchmod(f.fileno(),0755) - with open(here('src/uninstall.sh'),'wt') as f: - f.write(work.generate_uninstall_script()) - os.fchmod(f.fileno(),0755) - - for name,test in work.iter_tests(): - with open(here(test.j_job_name(work,con)),'wt') as f: - f.write(test.j_job_contents(work,con)) - os.fchmod(f.fileno(),0755) - with open(here(test.ex_script_name(work,con)),'wt') as f: - f.write(test.ex_script_contents(work,con)) - os.fchmod(f.fileno(),0755) - - with open(here('ush/functions.bash'),'wt') as f: - f.write('# DO NOT EDIT THIS SCRIPT; IT IS AUTOMATICALLY GENERATED\n') - f.write('# These are bash functions used by the ex-scripts.\n\n') - f.write(bash_functions) - -######################################################################## - -##@var _HAVE_NOT_PEEKED -# Special constant used by peekable to indicate nothing has been peeked yet. -# @warning Terrible things will happen if you overwrite this. -# @private -_HAVE_NOT_PEEKED=object() - -class peekable(object): - def __init__(self,iterator): - self.__child=iterator - self.__iterator=iter(iterator) - self.__peek=_HAVE_NOT_PEEKED - @property - def child(self): - return self.__child - def next(self): - if self.__peek is not _HAVE_NOT_PEEKED: - p,self.__peek = self.__peek,_HAVE_NOT_PEEKED - else: - p=self.__iterator.next() - return p - def peek(self): - if self.__peek is _HAVE_NOT_PEEKED: - self.__peek=self.__iterator.next() - return self.__peek - def at_end(self): - if self.__peek is not _HAVE_NOT_PEEKED: - return False - try: - self.__peek=self.__iterator.next() - except StopIteration as se: - return True - return False - def __iter__(self): - p,self.__peek = self.__peek,_HAVE_NOT_PEEKED - if p is not _HAVE_NOT_PEEKED: - yield p - for v in self.__iterator: - yield v - -######################################################################## - -class BaseObject(object): - def __init__(self,defscopes): - self.defscopes=defscopes - self.is_scope=False - self.is_filters=False - self.is_criteria=False - self.is_scalar=False - self.can_be_used=False - def bash_context(self,con): - raise Exception("Cannot express null_value in a bash string.") - def is_valid_rvalue(self,con): return True - def string_context(self,con): return "null" - def logical_context(self,con): return False - def numeric_context(self,con): return 0.0 - def _apply_rescope(self,scopemap=None,prepend=None): - if not prepend: prepend=[] - if not scopemap: scopemap={} - self.defscopes=prepend+[ scopemap[s] if s in scopemap else s - for s in self.defscopes ] - def rescope(self,scopemap=None,prepend=None): - if not prepend: prepend=[] - if not scopemap: scopemap={} - return BaseObject(prepend+[ scopemap[s] if s in scopemap else s - for s in self.defscopes ]) - def run(self,con): - print self.string_context(con) - def iterdeps(self): - return - yield 'a' # Syntactic trick to ensure this is an iterator. - -##@var null_value -# A special constant that indicates a variable without a value. -# @warning Terrible things will happen if you overwrite this. -null_value=BaseObject([]) - -######################################################################## - -class TypelessObject(BaseObject): - """!Represents an object that cannot be evauated in any context. - This is a convenience class intended to be used by subclasses to - disable all but certain contexts.""" - def bash_context(self,con): - raise TypeError('Cannot evaluate %s in a bash context.'%( - type(self).__name__,)) - def string_context(self,con): - raise TypeError('Cannot evaluate %s in a string context.'%( - type(self).__name__,)) - def logical_context(self,con): - raise TypeError('Cannot evaluate %s in a logical context.'%( - type(self).__name__,)) - def numeric_context(self,con): - raise TypeError('Cannot evaluate %s in a numeric context.'%( - type(self).__name__,)) - def run(self,con): - raise TypeError('Cannot run objects of type %s.'%( - type(self).__name__,)) - -######################################################################## - -class Scope(BaseObject): - def __init__(self,defscopes): - super(Scope,self).__init__(defscopes) - self.__vars=dict() - self.__parameters=dict() - self.is_scope=True - self.can_be_used=True - - def validate_parameter(self,name): pass - - def new_empty(self): return Scope(self.defscopes) - - def bash_context(self,con): - raise Exception("Cannot express a hash in a bash string.") - - def string_context(self,con): - if self.haslocal('_as_string'): - value=self.getlocal('_as_string') - return value.string_context(con) - else: - return str(id(self)) - - def no_nulls(self): - for k,v in self.iterlocal(): - if v is null_value: - return False - return True - - def _set_parameters(self,update): - self.__parameters.update(update) - for p in self.__parameters.iterkeys(): - self.validate_parameter(p) - - def numeric_context(self,con): - return len(self.__vars) + len(self.__parameters) - def logical_context(self,con): - return bool(self.__vars) or bool(self.__parameters) - - def as_parameters(self,con): - """!Changes all variables to parameters.""" - self.__parameters.update(self.__vars) - self.__vars=dict() - for p in self.__parameters.iterkeys(): - self.validate_parameter(p) - return self - - def rescope(self,scopemap=None,prepend=None): - scope=self.new_empty() - scope._apply_rescope(scopemap,prepend) - for k,v in self.__parameters.iteritems(): - scope.__parameters[k]=v.rescope(scopemap,prepend) - for k,v in self.__vars.iteritems(): - scope.__vars[k]=v.rescope(scopemap,prepend) - return scope - - def has_parameters(self): - return bool(self.__parameters) - - def use_from(self,used_scope,only_scalars=False): - if used_scope.has_parameters(): - raise Exception('Cannot "use" a function. Fixme: use better exception here.') - if not used_scope.is_scope: - raise Exception('Target of "use" statement is not a scope.') - if not used_scope.can_be_used: - raise Exception('Target of "use" statement cannot be used.') - prepend_me=[ self ] - found_non_scalars=False - for k,v in used_scope.iterlocal(): - if only_scalars and v.is_scope: - found_non_scalars=True - self.force_define(k,v.rescope({used_scope:self})) - return found_non_scalars - def apply_parameters(self,scope,con): - assert(not scope.__parameters) - assert(self.__parameters) - if not self.__parameters: - return self - s=self.new_empty() - yell('APPLY PARAMETERS from %s to %s type %s\n'%( - repr(scope),repr(self),type(s).__name__)) - for k,v in self.__parameters.iteritems(): - if scope.haslocal(k): - s.__vars[k]=scope.getlocal(k).rescope({self:s, scope:s}) - elif v is not null_value: - s.__vars[k]=v.rescope({self:s, scope:s}) - else: - raise Exception('%s: no argument sent for this parameter'%( - k,)) - for k,v in self.__vars.iteritems(): - if k not in s.__vars: - s.__vars[k]=v.rescope({self:s, scope:s}) - yell('RESULT IS %s %s\n'%( - type(s).__name__,repr(s))) - return s - - def __str__(self): - return '{' + ','.join( [ - "%s=%s"%(str(s),repr(k)) for s,k in self.iterlocal() - ] ) + '}' - - def __repr__(self): - return '{' + ','.join( [ - "%s=%s"%(str(s),repr(k)) for s,k in self.iterlocal() - ] ) + '}' - - def subscope(self,key): - """!Returns a Scope within this Scope, with the given name. - - @param key a valid identifier within this scope - @returns a Scope with the given name, within this Scope - @raise ValueError if the key contains a "%" - @raise TypeError if the key refers to something in this Scope - that is not a Scope. This is detected through the is_scope - attribute or property.""" - if "%" in key: - raise ValueError("Key \"%s\" is not a valid identifier"%(key,)) - if key in self.__parameters: - value=self.__parameters[key] - else: - value=self.__vars[key] - try: - if value.is_scope: - return value - except AttributeError as ae: - pass # value does not define is_scope - raise TypeError("Key \"%s\" refers to something that is not a Scope." - %(key,)) - - def getlocal(self,key): - """!Return the value of a key local to this scope without - searching other scopes. Will raise ValueError if the key - contains a "%" - - @param key a valid identifier within this scope - @returns The value of the key from this scope. - @raise ValueError if the key is syntactically not a valid identifier - such as one that contains a "%" - @raise KeyError if the key is a valid identifier but is not - within this scope.""" - if "%" in key: - raise ValueError("Key \"%s\" is not a valid identifier"%(key,)) - if key in self.__parameters: - return self.__parameters[key] - elif key in self.__vars: - return self.__vars[key] - raise KeyError(key) - - def setlocal(self,key,value): - """!Sets the value of a key within this scope. - - @param key a valid identifier to set within this scope - @param valuel the value of the identifier - @raise ValueError if the key is not a valid identifier, such as - one that contains a "%" """ - if '%' in key: - raise ValueError("Key \"%s\" contains a \"%\""%(key,)) - if key in self.__parameters: - raise Exception("Key \"%s\" is already a parameter. FIXME: Need better exception class."%(key,)) - self.__vars[key]=value - return value - - def haslocal(self,key): - return key in self.__parameters or key in self.__vars - - def iterlocal(self): - for k,v in self.__parameters.iteritems(): - yield k,v - for k,v in self.__vars.iteritems(): - yield k,v - - def resolve(self,key,scopes=None): - assert(isinstance(key,basestring)) - names=splitkey(key) - if scopes is None: - search=[self]+self.defscopes - else: - search=[self]+scopes - #scopestack=list() - yell('search for %s = %s in %s\n'%(repr(key),repr(names),repr(search))) - for name in names: - found=None - for scope in search: - try: - found=scope.getlocal(name) - #scopestack.insert(0,scope) - break # Done searching scopes. - except KeyError as ke: - yell('Key %s not in scope %s from top %s\n'%( - repr(name),repr(scope),repr(self))) - continue # Check for name in next scope. - if found is None: - raise KeyError(key) - search=[found] - if found is None: - raise KeyError(key) - # if subscopes: - # return ( found, scopestack ) - # else: - return found - - def force_define(self,key,value): - names=splitkey(key) - lval=self - for i in xrange(len(names)-1): - lval=lval.getlocal(names[i]) - lval.setlocal(names[-1],value) - return value - - def check_define(self,key,value): - names=splitkey(key) - lval=self - for i in xrange(len(names)-1): - lval=lval.subscope(names[i]) - if lval.haslocal(value): - raise Exception('Key %s already exists. (FIXME: Put better ' - 'exception class here.)'%(key,)) - #yell('setlocal %s = %s\n'%(names[-1],value)) - lval.setlocal(names[-1],value) - return value - - def expand_string(self,string,con,scopes=None): - stream=StringIO.StringIO() - yell('Expand %s in %s\n'%(repr(string),repr(self))) - def streamwrite(s): - #yell("Append \"%s\" to output string.\n"%(s,)) - stream.write(s) - for m in re.finditer(r'''(?sx) - ( - (?P[^@]+) - | @ \[ (?P@ ) \] - | @ \[ ' (?P [^']+ ) ' \] - | @ \[ (?P[^\]]+) \] - | (?P@ ) (?! \[ ) - | (?P.) - )''',string): - if m.group('text'): - streamwrite(m.group()) - elif m.group('escaped_text'): - streamwrite(m.group('escaped_text')) - elif m.group('escaped_at'): - streamwrite(m.group('escaped_at')) - elif m.group('literal_at'): - streamwrite(m.group('literal_at')) - elif m.group('varexpr'): - streamwrite(self.resolve(m.group('varexpr'),scopes) \ - .string_context(con)) - else: - raise ValueError("Parser error: invalid character \"%s\" in" - " \"%s\"\n"%(m.group(0),string)) - val=stream.getvalue() - stream.close() - return val - #print "Arg \"%s\" expands to \"%s\""%(string,val) - -def make_params(defscopes,**kwargs): - s=Scope(defscopes) - for k,v in kwargs.iteritems(): - s.setlocal(k,v) - return s.as_parameters() - -def make_scope(defscopes,**kwargs): - s=Scope(defscopes) - for k,v in kwargs.iteritems(): - s.setlocal(k,v) - return s - -def call_scope(scope,con,defscopes,**kwargs): - parms=make_scope(defscopes,**kwargs) - assert(parms.no_nulls()) - s=scope.apply_parameters(parms,con) - assert(s.no_nulls()) - return s - -######################################################################## - -class Builtin(Scope): - def __init__(self,defscopes,opname): - super(Builtin,self).__init__(defscopes) - self.__opname=opname - def bash_context(self,con): - raise TypeError('Cannot evaluate built-in operator %s in a ' - 'bash context.'%(self.__opname,)) - def string_context(self,con): - raise TypeError('Cannot evaluate built-in operator %s in a ' - 'string context.'%(self.__opname,)) - def logical_context(self,con): - raise TypeError('Cannot evaluate built-in operator %s in a ' - 'logical context.'%(self.__opname,)) - def numeric_context(self,con): - raise TypeError('Cannot evaluate built-in operator %s in a ' - 'numeric context.'%(self.__opname,)) - def run(self,con): - raise TypeError('Cannot run built-in operator %s.'%(self.__opname,)) - - def new_empty(self): - return Builtin(self.defscopes,self.__opname) - -######################################################################## - -class Copy(Builtin): - def __init__(self,defscopes,empty=False): - super(Copy,self).__init__(defscopes,'.copy.') - if not empty: - self._set_parameters({'src':null_value, 'tgt':null_value}) - def new_empty(self): - return Copy(self.defscopes,empty=True) - def run(self,con): - assert(self.no_nulls()) - src=self.resolve('src').string_context(con) - tgt=self.resolve('tgt').string_context(con) - produtil.fileop.deliver_file(src,tgt) - def bash_context(self,con): - assert(self.no_nulls()) - src=self.resolve('src').bash_context(con) - tgt=self.resolve('tgt').bash_context(con) - return 'deliver_file %s %s\n'%(src,tgt) - -######################################################################## - -class CopyDir(Builtin): - def __init__(self,defscopes,empty=False): - super(CopyDir,self).__init__(defscopes,'.copydir.') - if not empty: - self._set_parameters({'src':null_value, 'tgt':null_value}) - def new_empty(self): - return CopyDir(self.defscopes,empty=True) - def run(self,con): - raise NotImplementedError('CopyDir.run is not implemented yet.') - def bash_context(self,con): - assert(self.no_nulls()) - src=self.resolve('src').bash_context(con) - tgt=self.resolve('tgt').bash_context(con) - return '''for srcfile in %s/* ; do - deliver_file "$srcfile" %s/. -done -'''%(src,tgt) - -######################################################################## - -class Link(Builtin): - def __init__(self,defscopes,empty=False): - super(Link,self).__init__(defscopes,'.link.') - if not empty: - self._set_parameters({'src':null_value, 'tgt':null_value}) - def new_empty(self): - return Link(self.defscopes,empty=True) - def run(self,con): - assert(self.no_nulls()) - src=self.resolve('src').string_context(con) - tgt=self.resolve('tgt').string_context(con) - produtil.fileop.make_symlink(src,tgt) - def bash_context(self,con): - assert(self.no_nulls()) - src=self.resolve('src').bash_context(con) - tgt=self.resolve('tgt').bash_context(con) - return 'ln -sf %s %s\n'%(src,tgt) - -######################################################################## - -class AtParse(Builtin): - def __init__(self,defscopes,empty=False): - super(AtParse,self).__init__(defscopes,'.atparse.') - if not empty: - self._set_parameters({'src':null_value, 'tgt':null_value}) - def new_empty(self): - return AtParse(self.defscopes,empty=True) - def run(self,con): - raise NotImplementedError("FIXME: Sam has not implemented AtParse.run") - def bash_context(self,con): - out=StringIO.StringIO() - src=self.resolve('src').bash_context(con) - tgt=self.resolve('tgt').bash_context(con) - out.write("echo input to atparse from %s:\ncat %s\n"%(src,src)) - out.write("echo send to %s\n"%(tgt,)) - out.write("cat %s | atparse \\\n"%(src,)) - seen=set() - for scope in self.defscopes: - for k,v in scope.iterlocal(): - if k in seen: continue - seen.add(k) - if '%' in k or '.' in k: - pass#out.write('# $%s: skip; invalid shell variable name\n'%(k,)) - elif k[0:2] == '__': - pass#out.write("# $%s: skip; name begins with __\n"%(k,)) - elif v is null_value: - pass#out.write('# $%s: skip; has no value\n'%(k,)) - elif not v.is_scalar: - pass#out.write('# $%s: skip; value is not scalar\n'%(k,)) - else: - out.write(' %s=%s \\\n'%(k,v.bash_context(con))) - out.write(" > %s\n"%(tgt,)) - out.write("set -xe\n") - ret=out.getvalue() - out.close() - return ret - -######################################################################## - -class BitCmp(Builtin): - def __init__(self,defscopes,empty=False): - super(BitCmp,self).__init__(defscopes,'.bitcmp.') - if not empty: - self._set_parameters({'src':null_value, 'tgt':null_value}) - def new_empty(self): - return BitCmp(self.defscopes,empty=True) - def run(self,con): - src=self.resolve('src').string_context(con) - tgt=self.resolve('tgt').string_context(con) - if run_mode==BASELINE: - produtil.fileop.deliver_file(src,tgt) - return - if os.path.samefile(src,tgt): - # Same file object in filesystems. - return True - with open(src,'rt') as srcf: - with open(tgt,'rt') as tgtf: - srcstat=os.fstat(src.fileno()) - tgtstat=os.fstat(tgt.fileno()) - if not srcstat: return False # file stopped existing - if not tgtstat: return False # file stopped existing - if srcstat.st_size!=tgtstat.st_size: - # Different size according to stat - return False - eof=False - while not eof: - srcdat=src.read(1048576) - tgtdat=tgt.read(1048576) - if len(srcdat)!=len(tgtdat): - return False # Lengths differ - if srcdat!=tgtdat: - return False # Contents differ - eof=not len(srcdat) or not len(tgtdat) - return True - def bash_context(self,con): - src=self.resolve('src').bash_context(con) - tgt=self.resolve('tgt').bash_context(con) - if run_mode==BASELINE: - return 'deliver_file %s %s\n'%(tgt,src) - else: - return 'bitcmp %s %s\n'%(src,tgt) - -######################################################################## - -class Criteria(TypelessObject): - def __init__(self,defscopes): - super(Criteria,self).__init__(defscopes) - self.__opmap=collections.defaultdict(list) - self.__tgtlist=list() - self.is_criteria=True - def add_binary_operator(self,tgt,op,src,con): - if tgt not in self.__opmap: - self.__tgtlist.append(tgt) - callme=call_scope(op,con,self.defscopes,tgt=tgt,src=src) - for mycall in self.__opmap[tgt]: - assert mycall is not null_value - if mycall==callme: - return - callme=self.__opmap[tgt].append(callme) - assert callme is not null_value - def use_from(self,criteria,only_scalars=False): - if only_scalars: - raise ValueError('In Criteria.use_from, only_scalars must ' - 'be False.') - if not criteria.is_criteria: - raise TypeError('Criteria blocks can only use criteria blocks.') - for tgt,callme in criteria.itercriteria(): - found=False - for mycall in self.__opmap[tgt]: - if callme==mycall: - found=True - break - if not found: - self.__opmap[tgt].append(callme) - def itercriteria(self): - for tgt in self.__tgtlist: - for callme in self.__opmap[tgt]: - yield tgt,callme - def bash_context(self,con): - out=StringIO.StringIO() - for tgt in self.__tgtlist: - out.write('echo criteria for target %s:\n'%( - tgt.bash_context(con),)) - for callme in self.__opmap[tgt]: - out.write(callme.bash_context(con)) - if run_mode==BASELINE: break - ret=out.getvalue() - out.close() - return ret - -######################################################################## - -class Filters(TypelessObject): - def __init__(self,defscopes): - super(Filters,self).__init__(defscopes) - self.__opmap=dict() - self.__tgtlist=list() - self.is_filters=True - def add_binary_operator(self,tgt,op,src,con): - if tgt not in self.__opmap: - self.__tgtlist.append(tgt) - self.__opmap[tgt]=call_scope(op,con,self.defscopes, - tgt=tgt,src=src) - assert self.__opmap[tgt] is not null_value - def use_from(self,filters,only_scalars=False): - if only_scalars: - raise ValueError('In Filters.use_from, only_scalars must ' - 'be False.') - if not filters.is_filters: - raise TypeError('Filters blocks can only use filters blocks.') - for tgt,callme in filters.iterfilters(): - have_tgt=tgt in self.__opmap - if have_tgt and self.__opmap[tgt]==callme: - continue - self.__opmap[tgt]=callme - assert self.__opmap[tgt] is not null_value - if not have_tgt: - self.__tgtlist.append(tgt) - def iterfilters(self): - for tgt in self.__tgtlist: - yield tgt,self.__opmap[tgt] - def bash_context(self,con): - out=StringIO.StringIO() - for tgt in self.__tgtlist: - out.write('echo Filter for target %s:\n'%( - tgt.bash_context(con),)) - out.write(self.__opmap[tgt].bash_context(con)) - ret=out.getvalue() - out.close() - return ret - -######################################################################## - -class Rank(TypelessObject): - def __init__(self,args,opts): - self.__args=args - self.__opts=opts - @property - def args(self): - return self.__args - def argiter(self): - for arg in self.__args: - yield arg - def ranks(self,con): - if self.__opts is None or not self.__opts.haslocal('ranks'): - return 0 - return int(self.__opts.getlocal('ranks').numeric_context(con)) - def threads(self,con): - if self.__opts is None or not self.__opts.haslocal('threads'): - return 0 - return int(self.__opts.getlocal('threads').numeric_context(con)) - -######################################################################## - -class SpawnProcess(TypelessObject): - def __init__(self,defscopes): - super(SpawnProcess,self).__init__(defscopes) - self.__ranks=list() - def add_rank(self,args,opts): - self.__ranks.append(Rank(args,opts)) - def mpi_comm_size(self,con): - size=0 - for rank in self.__ranks: - size+=rank.ranks(con) - return size - def rocoto_resources(self,con): - size=max(int(self.mpi_comm_size(con)),2) - return '%d\n'%size - def bash_context(self,con): - out=StringIO.StringIO() - out.write('# Embedded process execution:\n') - need_ranks=len(self.__ranks)>1 - have_ranks=False - ranks=list() - threads=list() - for rank in self.__ranks: - nranks=rank.ranks(con) - if nranks>0: have_ranks=True - if nranks<1 and need_ranks: - nranks=1 - ranks.append(nranks) - threads.append(rank.threads(con)) - nthreads=max(0,threads[0]) - out.write('export OMP_NUM_THREADS=%d MKL_NUM_THREADS=0\n'%( - nthreads,)) - if not have_ranks: - # Serial or openmp program. - out.write(' '.join([r.bash_context(con) - for r in self.__ranks[0].args])) - out.write('\n') - elif len(self.__ranks)==1: - out.write('mpirun -np %d '%(int(self.__ranks[0].ranks(con)))) - out.write(' '.join([r.bash_context(con) - for r in self.__ranks[0].args])) - out.write('\n') - else: - raise Exception("FIXME: Sam has not implemented MPMD yet.") - out.write('# End of embedded process execution.\n') - ret=out.getvalue() - out.close() - return ret - -######################################################################## - -class EmbedBash(Scope): - def __init__(self,defscopes): - super(EmbedBash,self).__init__(defscopes) - self.__template=None - - def validate_parameter(self,name): - pass - #if not re.match('(?s)^[a-zA-Z][a-zA-Z0-9_]*$',name): - #raise ValueError('Invalid bash variable name $%s FIXME: use better exception here'%(name,)) - - def bash_context(self,con): - raise Exception("Cannot express a bash script in a bash string.") - - def __str__(self): - return "bash script \"%s\" %s"%( - elipses(repr(self.gettemplate())), - super(EmbedBash,self).__str__()) - - def __repr__(self): - return "bash script \"%s\" %s"%( - elipses(repr(self.gettemplate())), - super(EmbedBash,self).__str__()) - - def apply_parameters(self,scope,con): - s=super(EmbedBash,self).apply_parameters(scope,con) - s.__template=self.__template.rescope({self:s, scope:s}) - return s - - def is_valid_rvalue(self,con): - return self.__template is not None - - def string_context(self,con): - return '%d'%(self.numeric_context(con),) - - def settemplate(self,template): - assert(isinstance(template,String)) - self.__template=template - - def gettemplate(self): - return self.__template - - def numeric_context(self,con): - return self.run(con) - - def bash_context(self,con): - template=self.gettemplate() - template=template.string_context(con) - expanded=self.expand_string(template,con) - - stream=StringIO.StringIO() - env=dict() - unset_me=list() - for k,v in self.iterlocal(): - if '%' in k or '.' in k: - stream.write('# $%s: skip; invalid shell variable name\n' - %(k,)) - elif k[0:2] == '__': - stream.write("# $%s: skip; name begins with __\n"%(k,)) - elif v is null_value: - stream.write('# $%s: skip; has no value\n'%(k,)) - elif not v.is_scalar: - stream.write('# $%s: skip; value is not scalar\n'%(k,)) - else: - unset_me.append(k) - stream.write('%s=%s\n'%(k,v.bash_context(con))) - stream.write("# Embedded bash script:\n") - stream.write(expanded) - stream.write('\n# End of embedded bash script.\n') - for k in unset_me: - stream.write('unset %s\n'%(k,)) - stream.write('set -xe\n\n') - script=stream.getvalue() - stream.close() - return script - - def run(self,con): - script=self.bash_context(con) - yell('%-7s %-7s %s\n'%("RUN","BASH",script)) - cmd=produtil.run.exe("bash")<<'set -xue\n'+script - env=dict(self.iterlocal()) - if env: cmd.env(**env) - return produtil.run.run(cmd) - - def logical_context(self,con): - return bool(self.numeric_context(con)==0) - - def new_empty(self): - s=EmbedBash(self.defscopes) - s.__template=self.__template - return s - -######################################################################## - -class Task(Scope): - def __init__(self,defscopes,name,runvar='run'): - super(Task,self).__init__(defscopes) - self.__deps=list() - self.__name=str(name) - self.runvar=runvar - @property - def name(self): - return self.__name - def bash_context(self,con): - assert(self.haslocal(self.runvar)) - return self.getlocal(self.runvar).bash_context(con) - - def iterdeps(self): - for dep in self.__deps: - yield dep - - def add_dependency(self,dep): - self.__deps.append(dep) - - def is_valid_rvalue(self,con): - return self.haslocal(self.runvar) and \ - self.getlocal(self.runvar) is not null_value - - def string_context(self,con): - return self.getlocal(self.runvar).string_context(con) - - def numeric_context(self,scopes,con): - return self.getlocal(self.runvar).numeric_context(con) - - def run(self,con): - return self.getlocal(self.runvar).run(con) - - def logical_context(self,con): - return self.getlocal(self.runvar).numeric_context(con) - - def new_empty(self): - return Task(self.defscopes,self.__name,self.runvar) - -######################################################################## - -class Build(Task): - def __init__(self,defscopes,name): - super(Build,self).__init__(defscopes,name,'build') - def new_empty(self): - return Build(self.defscopes,self.name) - -######################################################################## - -class Platform(Task): - def __init__(self,defscopes,name): - super(Platform,self).__init__(defscopes,name,'detect') - def new_empty(self): - return Platform(self.defscopes,self.name) - -######################################################################## - -class Test(Scope): - def __init__(self,defscopes,name,mode): - super(Test,self).__init__(defscopes) - assert(mode in [ BASELINE, EXECUTION ]) - self.mode=mode - self.__name=str(name) - self.__deps=list() - - @property - def name(self): - return self.__name - - def bash_context(self,con): - if self.mode==BASELINE: - steps=['prep','input','execute','make_baseline'] - else: - steps=['prep','input','execute','verify'] - - out=StringIO.StringIO() - for step in steps: - try: - stepobj=self.getlocal(step) - except KeyError as ke: - if step in [ 'make_baseline', 'verify' ]: - stepobj=self.getlocal('output') - else: - raise - out.write(stepobj.bash_context(con)) - out.write('\n') - ret=out.getvalue() - out.close() - return ret - - def iterdeps(self): - for dep in self.__deps: - yield dep - - def add_dependency(self,dep): - self.__deps.append(dep) - - def is_valid_rvalue(self,con): - if self.mode==BASELINE: - steps=['prep','input','execute','make_baseline'] - else: - steps=['prep','input','execute','verify'] - - for step in steps: - if self.haslocal(step): - if self.getlocal(step) is not null_value: - continue - elif step in ['make_baseline','verify'] and \ - self.haslocal('output'): - if self.getlocal('output') is not null_value: - continue - raise KeyError(step) - return True - - def string_context(self,con): - raise TypeError('A Test cannot be evaluated in a string context.') - - def numeric_context(self,con): - raise TypeError('A Test cannot be evaluated in a numeric context.') - - def run(self,con): - raise TypeError('A Test cannot be run directly.') - - def logical_context(self,con): - raise TypeError('A Test cannot be evaluated in a logical context.') - - def new_empty(self): - return Test(self.defscopes,self.name,self.mode) - -######################################################################## - -class AutoDetectPlatform(object): - def __init__(self): - super(AutoDetectPlatform,self).__init__() - self.__platforms=list() - def add(self,platform): - self.__platforms.append(platform) - def detect(self,con): - matches=list() - names=list() - for platform in self.__platforms: - detecter=platform.resolve('detect') - name=platform.resolve('PLATFORM_NAME') - name=name.string_context(con) - sys.stderr.write('%s: detection...\n'%(name,)) - detection=detecter.logical_context(con) - if detection: - sys.stderr.write('%s: PLATFORM DETECTED\n'%(name,)) - matches.append(platform) - names.append(name) - else: - sys.stderr.write('%s: not detected\n'%(name,)) - sys.stderr.write('List of platforms detected: '+ - ' '.join([ repr(s) for s in names ])+'\n') - return matches - -######################################################################## - -class Numeric(BaseObject): - def __init__(self,value): - super(Numeric,self).__init__([]) - self.is_scalar=True - self.__value=value - def string_context(self,con): - return '%g'%self.__value - def bash_context(self,con): - return '"%g"'%self.__value - def numeric_context(self,con): - return self.__value - def logical_context(self,con): - return self.numeric_context(con)!=0 - def __str__(self): - return str(self.__value) - def __repr__(self): - return repr(self.__value) - def rescope(self,scopemap=None,prepend=None): - return Numeric(self.__value) - -######################################################################## - -class String(BaseObject): - def __init__(self,defscopes,value,should_expand): - super(String,self).__init__(defscopes) - self.__value=str(value) - self.is_scalar=True - self.should_expand=bool(should_expand) - def rescope(self,scopemap=None,prepend=None): - if prepend is None: prepend=[] - if scopemap is None: scopemap={} - return String(prepend+[ scopemap[s] if s in scopemap else s - for s in self.defscopes ], - self.__value,self.should_expand) - def string_context(self,con): - if self.should_expand: - return self.defscopes[0].expand_string( - self.__value,con,self.defscopes[1:]) - else: - return self.__value - def bash_context(self,con): - string=self.string_context(con) - output=StringIO.StringIO() - for m in re.finditer('''(?xs) - ( - (?P'+) - | (?P[!-&(-\[\]-~ ]+) - | (?P.) - )''',string): - if m.group('quotes'): - output.write('"' + m.group('quotes') + '"') - elif m.group('printable'): - output.write("'"+m.group('printable')+"'") - elif m.group('control'): - output.write("$'\%03o'"%ord(m.group('control'))) - ret=output.getvalue() - output.close() - return ret - def logical_context(self,con): - s=self.string_context(con) - s=s[-30:].lower() - if s in [ '.true.', 'true', 'yes', 't', 'y' ]: return True - if s in [ '.false.', 'false', 'no', 'f', 'n' ]: return False - try: - i=float(s) - except ValueError as ve: - pass - raise ValueError('Cannot parse %s as a logical value.'%(s,)) - def numeric_context(self,con): - s=self.string_context(con) - return float(s) - def __str__(self): return self.__value - def __repr__(self): return 'String(%s)'%(repr(self.__value),) - -######################################################################## - -class Environ(Scope): - def __init__(self): - super(Environ,self).__init__([]) - self.can_be_used=False - def new_empty(self): return Environ() - def bash_context(self,con): - raise Exception('Cannot express the environment in a bash context.') - def string_context(self,con): - raise Exception('Cannot evaluate the environment in a string context.') - def no_nulls(self): return True - def _set_parameters(self,update): - raise Exception('Cannot set parameters in the environment.') - def numeric_context(self,con): - raise Exception('Cannot evaluate the environment in a numeric context.') - def logical_context(self,con): - raise Exception('Cannot evaluate the environment in a logical context.') - def as_parameters(self,con): - raise Exception('Cannot turn the environment into a parameter list.') - def rescope(self,scopemap=None,prepend=None): - return self - def has_parameters(self): - return False - def use_from(self,used_scope,only_scalars=False): - raise Exception('Cannot use other scopes within the environment.') - def apply_parameters(self,scope,con): - raise Exception('Cannot call the environment.') - def __str__(self): - return 'Environ()' - def __repr__(self): - return 'Environ()' - def subscope(self,key): - raise TypeError('The environment has no subscopes.') - def getlocal(self,key): - return String([self],os.environ[key],False) - def setlocal(self,key,value): - raise Exception('Refusing to modify the environment.') - def haslocal(self,key): - return key in os.environ - def iterlocal(self): - for k,v in os.environ.iteritems(): - yield k,String([self],v,False) - def resolve(self,key,scopes=None): - if '.' in key or '%' in key: - raise ValueError('Invalid environment variable \"%s\".'%(key,)) - return os.environ[key] - def force_define(self,key,value): - raise Exception('Refusing to modify the environment.') - def check_define(self,key,value): - raise Exception('Refusing to modify the environment.') - -######################################################################## - -unknown_file='(**unknown**)' - -class Token(object): - def __init__(self,token_type,token_value,filename,lineno): - super(Token,self).__init__() - self.token_type=token_type - self.filename=filename - self.lineno=lineno - self.token_value=token_value - def __repr__(self): - return 'Token(%s,%s,%s,%s)'%( - repr(self.token_type),repr(self.token_value), - repr(self.filename),repr(self.lineno)) - def __str__(self): - return 'Token(%s,%s,%s,%s)'%( - repr(self.token_type),repr(self.token_value), - repr(self.filename),repr(self.lineno)) - -##@var end_of_line_type -# The token_type parameter to send to Token.__init__() to indicate the -# end of a line -end_of_line_type='\n' - -##@var end_of_text_type -# The token_type parameter to send to Token.__init__() to indicate the -# end of a file or string. -end_of_text_type='' - -class Tokenizer(object): - def copy(self): - return Tokenizer() - def __init__(self): - super(Tokenizer,self).__init__() - #yell('compile\n') - self.re=re.compile(r'''(?xs) - ( - \# (?P[^\r\n]+) (?: \r | \n )+ - | \# (?P[^\r\n]+) $ - | (?P [A-Za-z_] [A-Za-z_0-9.]* - (?: % [A-Za-z_][A-Za-z_0-9.]* )* ) - | (?P - [+-]? [0-9]+\.[0-9]+ (?: [eE] [+-]? [0-9]+ )? - | [+-]? \.[0-9]+ (?: [eE] [+-]? [0-9]+ )? - | [+-]? [0-9]+\. (?: [eE] [+-]? [0-9]+ )? - | [+-]? [0-9]+ (?: [eE] [+-]? [0-9]+ )? - ) - | ' (?P (?: - [^'\\] - | ( \\ . )+ ) * ) ' - | " (?P (?: - [^"\\] - | ( \\ . )+ ) * ) " - | \[\[\[ (?P (?: - [^\]@] - | @ (?!\[) - | @ \[ @ \] - | @ \[ ' [^']+ ' \] - | @ \[ [^\]]+ \] - | \]\] (?!\]) - | \] (?!\]) - ) *? ) \]\]\] - | (?P[ \t]* [\r\n]+) - | (?P = ) - | (?P \* ) - | (?P [ \t]+ ) - | (?P\{) - | (?P\}) - | (?P\(/) - | (?P/\)) - | (?P\() - | (?P\)) - | (?P,) - | (?P:) - | (?P\.[a-zA-Z_][a-zA-Z0-9_.]*\.) - | (?P . ) - )''') - def tokenize(self,text,filename=unknown_file,first_line=1): - lineno=first_line - for m in self.re.finditer(text): - if m is None: - raise ValueError('SHOULD NOT GET HERE: no match on "%s"'%(line,)) - # else: - # for dkey,dval in m.groupdict().iteritems(): - # if dval is not None: - # yell("%10s = %s\n"%(dkey,repr(dval))) - if m.group('comment'): - yield Token(end_of_line_type,m.group('comment'), - filename,lineno) - elif m.group('commentend'): - yield Token(end_of_line_type,m.group('commentend'), - filename,lineno) - elif m.group('endline'): - yield Token(end_of_line_type,m.group('endline'), - filename,lineno) - elif m.group('oper'): - yield Token('oper',m.group('oper'),filename,lineno) - elif m.group('varname'): - yield Token('varname',m.group('varname'),filename,lineno) - elif m.group('number'): - yield Token('number',m.group('number'),filename,lineno) - elif m.group('qstring'): - yield Token('qstring',m.group('qstring'),filename,lineno) - elif m.group('dqstring'): - yield Token('dqstring',m.group('dqstring'),filename,lineno) - elif m.group('bracestring'): - yield Token('bracestring',m.group('bracestring'), - filename,lineno) - elif m.group('equal'): - yield Token('=','=',filename,lineno) - elif m.group('comma'): - yield Token(',',',',filename,lineno) - elif m.group('colon'): - yield Token(':',':',filename,lineno) - elif m.group('lset'): - yield Token('{','{',filename,lineno) - elif m.group('rset'): - yield Token('}','}',filename,lineno) - elif m.group('lparen'): - yield Token('(','(',filename,lineno) - elif m.group('rparen'): - yield Token(')',')',filename,lineno) - elif m.group('lfort'): - yield Token('(/','(/',filename,lineno) - elif m.group('rfort'): - yield Token('/)','/)',filename,lineno) - elif m.group('whitespace'): - pass # Ignore whitespace outside strings - else: - raise ValueError('%s:%d: invalid text \"%s\"'%( - filename,lineno,m.group(0))) - lineno+=m.group(0).count('\n') - yield Token(end_of_text_type,'',filename,lineno) - -class TokenizeFile(object): - def __init__(self,tokenizer,fileobj,filename=unknown_file,first_line=1): - self.tokenizer=tokenizer - self.fileobj=fileobj - self.filename=filename - self.first_line=first_line - def for_file(self,fileobj,filename,first_line=1): - return TokenizeFile(self.tokenizer.copy(),fileobj,filename,first_line) - def __iter__(self): - text=self.fileobj.read() - for token in self.tokenizer.tokenize( - text,self.filename,self.first_line): - yield token - -######################################################################## - -class Parser(object): - def __init__(self,run_mode=None,logger=None): - super(Parser,self).__init__() - if logger is None: logger=module_logger - if run_mode is None: run_mode=EXECUTION - if run_mode is not EXECUTION and run_mode is not BASELINE: - raise ValueError( - 'The Parser.__init__ run_mode argument must be the ' - 'special module constants EXECUTION or BASELINE.') - self.__runset=set() - self.__runlist=list() - self.__run_mode=run_mode - self.__logger=logger - def iterrun(self): - for runme,con in self.__runlist: - yield runme,con - def con(self,token,scopes=None): - if scopes is None: scopes=[] - return Context(scopes,token,self.__run_mode,self.__logger) - def add_run(self,runme,con): - assert(isinstance(runme,BaseObject)) - if runme in self.__runset: - return # runme is already in the runlist - addlist=[ [runme,con] ] - addset=set([runme]) - for prereq in runme.iterdeps(): - if prereq==runme: - raise Exception('ERROR: %s is dependent on itself.'%( - repr(prereq),)) - if prereq in self.__runset or prereq in addlist: - continue # skip things we already ran. - assert(isinstance(prereq,BaseObject)) - addset.add(prereq) - addlist.append([prereq,con]) - for r in addlist: - assert(isinstance(r,list)) - assert(len(r)==2) - assert(isinstance(r[0],BaseObject)) - assert(isinstance(r[1],Context)) - self.__runset.update(addset) - self.__runlist.extend([r for r in reversed(addlist)]) - def parse(self,tokenizer,scope): - tokiter=peekable(tokenizer) - scope.setlocal('ENV',Environ()) - scope.setlocal('UNIQUE_ID',Numeric(os.getpid())) - try: - return self.parse_subscope( - tokiter,[scope],[end_of_text_type], - self.parse_between_assignments, - allow_overwrite=False, - allow_resolve=True, - allow_run=True, - allow_null=False, - allow_use=False, - allow_load=True, - scope_name='global scope') - except Exception as e: # FIXME: change exception type - # try: - # peek=tokiter.peek() - # filename=peek.filename - # lineno=peek.lineno - # sys.stderr.write('%s:%d: uncaught exception: %s\n'%( - # filename,lineno,str(e))) - # except StopIteration as se: - # sys.stderr.write('StopIteration while peeking: %s\n'%( - # str(se),)) - # pass - raise - def parse_between_arguments(self,tokiter,ends=None): - if ends is None: ends=[')'] - peek=tokiter.peek() - yell('%-7s peek type=%s value=%s\n'%( - 'BETWEEN',str(peek.token_type),elipses(str( - peek.token_value)))) - while True: - if peek.token_type == ',': - tokiter.next() - yell('%-7s consume ,\n'%('BETWEEN',)) - return True - elif peek.token_type in ends: - yell('%-7s stop at %s\n'%('BETWEEN',peek.token_type)) - return False - elif peek.token_type==end_of_line_type: - yell('%-7s saw \n'%('BETWEEN',)) - tokiter.next() - peek=tokiter.peek() - else: - return False - def parse_between_assignments(self,tokiter): - peek=tokiter.peek() - seen=False - yell('%-7s peek type=%s value=%s in parse_between_assignments\n'%( - 'BETWEEN',str(peek.token_type), - elipses(str(peek.token_value)))) - while True: - seen=True - if peek.token_type in [ ',', ';' ]: - tokiter.next() - yell('%-7s consume %s\n'%('BETWEEN',peek.token_type)) - return True - elif peek.token_type in [ '}', end_of_text_type ]: - yell('%-7s stop at %s\n'%('BETWEEN',peek.token_type)) - return True - elif peek.token_type==end_of_line_type: - yell('%-7s saw %s\n'%('BETWEEN',repr(peek.token_type))) - seen=True - tokiter.next() - peek=tokiter.peek() - else: - break - return seen - self.error('between_assignments',peek) - - def parse_embed_script(self,tokiter,scopes,ends,parse_between=None): - token=tokiter.next() - if token.token_type != 'varname': - self.error('embed',token) - if token.token_value != 'bash': - self.error('embed',token,'unknown language "%s"'%( - token.token_value,)) - nametoken=tokiter.next() - if token.token_type != 'varname': - self.error('embed script name',token) - scope=EmbedBash(scopes) - token=tokiter.next() - - while token.token_type==end_of_line_type: token=tokiter.next() - if token.token_type=='(': - self.parse_subscope(tokiter,[scope]+scopes,[')'], - self.parse_between_arguments, - allow_overwrite=False, - allow_resolve=False, - allow_null=True, - only_scalars=True, - scope_name='embed script parameters') - scope=scope.as_parameters(self.con(token,scopes)) - token=tokiter.next() - while token.token_type==end_of_line_type: token=tokiter.next() - - if token.token_type=='{': - self.parse_subscope(tokiter,[scope]+scopes,['}'], - self.parse_between_assignments, - allow_overwrite=True, - allow_resolve=True, - allow_null=False, - allow_use=True, - only_scalars=True, - scope_name='embed script variables') - token=tokiter.next() - while token.token_type==end_of_line_type: token=tokiter.next() - - if token.token_type in [ 'qstring', 'dqstring', 'bracestring' ]: - scope.settemplate(self.action_string([scope]+scopes,token)) - else: - self.error('embed script contents',token) - if parse_between: - parse_between(tokiter) - return (nametoken.token_value,scope) - - def parse_deplist(self,tokiter,scopes,task,ends): - allscopes=[task]+scopes - peek=tokiter.peek() - while not peek.token_type in ends: - if peek.token_type=='varname': - varname=peek - tokiter.next() - peek=tokiter.peek() - if peek.token_type==end_of_line_type: - continue # ignore blank lines - elif self.parse_between_arguments(tokiter,['{']): - # varname is followed by a comma - dep=self.action_resolve(varname,allscopes) - self.action_dependency(task,scopes,dep) - peek=tokiter.peek() - continue - elif peek.token_type in ends: - dep=self.action_resolve(varname,allscopes) - self.action_dependency(task,scopes,dep) - return - elif peek.token_type == '(': - # This is a function call. - tokiter.next() - subscope=Scope(scopes) - self.parse_subscope(tokiter,scopes,[')'], - self.parse_between_arguments, - allow_overwrite=False, - allow_resolve=False, - allow_null=True, - scope_name='dependency argument list') - subscope=subscope.as_parameters(self.con(peek,scopes)) - peek=tokiter.peek() - if self.parse_between_arguments(tokiter) \ - or peek.token_type in ends: - dep=self.action_call(varname,peek,scopes,subscope) - self.action_dependency(task,scopes,dep) - peek=tokiter.peek() - continue - self.error('dependency argument list',peek) - def parse_op_list(self,tokiter,scopes,subscope): - token=tokiter.next() - strings=[ 'qstring', 'dqstring', 'bracestring' ] - if token.token_type != '{': - self.error('operation list',token) - while True: - # Get target of operation: - token=tokiter.next() - while token.token_type==end_of_line_type: - token=tokiter.next() - - if token.token_type=='varname' and token.token_value=='use': - peek=tokiter.peek() - if peek.token_type!='varname': - self.error('operation list use statement',peek) - tokiter.next() - subscope.use_from(self.action_resolve(peek,scopes)) - self.parse_between_assignments(tokiter) - peek=tokiter.peek() - if peek.token_type=='}': - tokiter.next() - return subscope - continue - elif token.token_type=='}': - return subscope - elif token.token_type not in strings: - self.error('operator target',token) - tgt=self.action_string(scopes,token) - - # Get operator: - token=tokiter.next() - while token.token_type==end_of_line_type: - token=tokiter.next() - if token.token_type!='oper': - self.error('oper',token) - op=self.action_operator(scopes,token) - - # Get source of operation (input or baseline file) - token=tokiter.next() - while token.token_type==end_of_line_type: - token=tokiter.next() - if token.token_type not in strings: - self.error('operator source (input or baseline)',token) - src=self.action_string(scopes,token) - - # Add operator: - # FIXME: CONTEXT - subscope.add_binary_operator( - tgt,op,src,self.con(token,scopes)) - - self.parse_between_assignments(tokiter) - peek=tokiter.peek() - if peek.token_type=='}': - tokiter.next() - return subscope - - - def parse_hash_define(self,tokiter,scopes,subscope,parse_between=None, - allow_deps=False,hash_type='hash'): - token=tokiter.next() - parameters=False - - # if token.token_type=='(': - # parameters=True - # self.parse_subscope(tokiter,[subscope]+scopes,[')'], - # self.parse_between_arguments, - # allow_overwrite=False, - # allow_resolve=False, - # allow_null=True, - # scope_name=hash_type+' argument list') - # subscope=subscope.as_parameters() - # token=tokiter.next() - - if allow_deps and token.token_type==':': - self.parse_deplist( - tokiter,[subscope]+scopes,subscope,['{']) - token=tokiter.next() - - if token.token_type=='{': - tokiter.next() - self.parse_subscope(tokiter,[subscope]+scopes,['}'], - self.parse_between_assignments, - allow_overwrite=True, - allow_resolve=True, - allow_null=False, - allow_use=True, - scope_name=hash_type+' definition') - else: - self.error( - hash_type+' definition',token, - 'missing {...} block in '+hash_type+' definition') - - if parse_between: - parse_between(tokiter) - - yell('%-7s define Scope@%s with %sparameters\n'%( - hash_type.upper(),str(id(subscope)), - ' ' if parameters else 'no ')) - return subscope - - def parse_spawn_element(self,tokiter,scopes,spawn,ends): - token=tokiter.next() - args=list() - opts=list() - saw_vars=False - strings=[ 'qstring', 'dqstring', 'bracestring' ] - while token.token_type not in ends: - if token.token_type in strings: - if saw_vars: - self.error('spawn process',token,'var=value elements ' - 'must come after all arguments') - args.append(token) - self.parse_between_arguments(tokiter,ends) - token=tokiter.next() - continue - elif token.token_type==end_of_line_type: - token=tokiter.next() - continue - elif token.token_type=='varname': - name=token.token_value - peek=tokiter.peek() - if peek.token_type != '=': - self.error('spawn process',token) - tokiter.next() - else: - self.error('spawn process',token) - # we're at the value in varname=value - rvalue=self.parse_rvalue( - tokiter,scopes,['}',','], - only_scalars=True) - self.parse_between_arguments(tokiter,ends) - opts.append([name,rvalue]) - token=tokiter.next() - scope=Scope(scopes) - allscopes=[scope]+scopes - for k,v in opts: - scope.setlocal(k,v) - if not args: - self.error('spawn process',token,'no command nor arguments') - for arg in args: - assert(isinstance(arg,Token)) - argobjs=[ - self.action_string(allscopes,arg) for arg in args] - return argobjs,scope - - def parse_spawn_block(self,tokiter,scopes,name,spawn,ends,parse_between): - token=tokiter.next() - while token.token_type==end_of_line_type: - token=tokiter.next() - while token.token_type not in ends: - if token.token_type!='{': - self.error('spawned process',token) - (args,opts)=self.parse_spawn_element( - tokiter,scopes,spawn,['}']) - spawn.add_rank(args,opts) - if parse_between: parse_between(tokiter) - token=tokiter.next() - - def parse_spawn(self,tokiter,scopes,name,spawn): - token=tokiter.next() - if token.token_type!='{': - self.error('spawn block',token) - while self.parse_spawn_block(tokiter,scopes,name,spawn,['}'], - self.parse_between_assignments): - continue - return spawn - - def parse_autodetect(self,tokiter,scopes,taskname,task): - # Check for the (/ and skip it: - token=tokiter.next() - while token.token_type==end_of_line_type: - token=tokiter.next() - if token.token_type!='(/': - self.error('autodetect platform list',token) - - while True: - peek=tokiter.peek() - while peek.token_type==end_of_line_type: - tokiter.next() - peek=tokiter.peek() - if peek.token_type=='/)': - tokiter.next() - return - rvalue=self.parse_rvalue(tokiter,scopes,['/)'], - self.parse_between_arguments,False) - task.add(rvalue) - peek=tokiter.peek() - self.parse_between_arguments(tokiter) - if peek.token_type=='/)': - tokiter.next() - return - - def parse_load(self,tokiter,scope,seen_run): - filetoken=tokiter.next() - if filetoken.token_type!='qstring': - self.error('load',token,"load statements can only include " - "'single-quote strings'") - eoln=tokiter.peek() - if eoln.token_type not in [ end_of_line_type, end_of_text_type ]: - self.error('load',eoln,"a load statement must be followed " - "by an end of line or the end of the script.") - newfile=filetoken.token_value - if os.path.isabs(newfile): - newfile=os.path.join(filetoken.filename,newfile) - tokenizer=tokiter.child - with open(newfile,'rt') as fileobj: - new_tokenizer=tokenizer.for_file(fileobj,newfile) - new_tokiter=peekable(new_tokenizer) - self.parse_subscope( - new_tokiter,[scope],[end_of_text_type], - self.parse_between_assignments, - allow_overwrite=False, - allow_resolve=True, - allow_run=True, - allow_null=False, - allow_use=False, - allow_load=True, - scope_name='global scope', - seen_run=seen_run) - - def parse_subscope(self,tokiter,scopes,ends,parse_between, - allow_overwrite=True,allow_resolve=True, - allow_run=False,allow_null=False, - allow_use=False,scope_name='subscope', - only_scalars=False,allow_load=False, - seen_run=False): - go=True # set to False once an "ends" is seen - seen_run=bool(seen_run) # Did we see an execution request yet? - token=None - strings=[ 'qstring', 'dqstring', 'bracestring' ] - def define(con,key,val): - if seen_run: - self.error( - scope_name,token, - reason='Definitions must come before execution ' - 'requests.') - if not val.is_valid_rvalue(con): # FIXME: con - self.error(scope_name,token,'not a valid rvalue: %s'%( - elipses(repr(val)),)) - yell('%s:%s: define %s=%s\n'%( - token.filename,str(token.lineno), - str(key),repr(val))) - if allow_overwrite: - scopes[0].force_define(key,val) - else: - scopes[0].check_define(key,val) - if allow_resolve: - search_scopes=scopes - else: - search_scopes=scopes[1:] - while go: - token=tokiter.next() - if token.token_type=='varname': - peek=tokiter.peek() - if peek.token_type=='=': - tokiter.next() - define(self.con(token,scopes),token.token_value, - self.parse_rvalue(tokiter,search_scopes,ends, - parse_between, - only_scalars=only_scalars)) - parse_between(tokiter) - continue - elif token.token_value=='load' and peek.token_type in strings: - if not allow_load: - self.error('subscope',token,'load statements are ' - 'only allowed in the global scope.') - self.parse_load(tokiter,scopes[-1],seen_run) - if parse_between: parse_between(tokiter) - continue - elif token.token_value=='use' and peek.token_type=='varname' \ - and allow_use: - tokiter.next() # consume the peeked value - self.action_use(scopes,peek, - only_scalars=only_scalars) - if parse_between: parse_between(tokiter) - continue - elif allow_run and token.token_value=='run' \ - and peek.token_type=='varname': - seen_run=True - self.action_run_obj( - self.parse_rvalue(tokiter,search_scopes, - ends,parse_between), - self.con(peek,scopes)) - if parse_between: parse_between(tokiter) - continue - elif not only_scalars and token.token_value=='spawn' \ - and peek.token_type=='varname': - taskname=peek.token_value - tokiter.next() - task=self.parse_spawn(tokiter,scopes,peek.token_value, - SpawnProcess(scopes)) - define(self.con(peek,scopes),taskname,task) - if parse_between: parse_between(tokiter) - del taskname,task - continue - elif not only_scalars and token.token_value in [ - 'filters', 'criteria' ] and peek.token_type=='varname': - taskname=peek.token_value - tokiter.next() - if token.token_value=='filters': - task=Filters(scopes) - elif token.token_value=='criteria': - task=Criteria(scopes) - task=self.parse_op_list(tokiter,scopes,task) - define(self.con(peek,scopes),taskname,task) - if parse_between: parse_between(tokiter) - del taskname,task - continue - elif not only_scalars and token.token_value=='autodetect' \ - and peek.token_type=='varname': - taskname=peek.token_value - tokiter.next() # Skip name token - task=AutoDetectPlatform() - self.parse_autodetect(tokiter,scopes,taskname,task) - task=self.action_autodetect(self.con(peek,scopes), - tokiter,scopes,taskname,task) - define(self.con(peek,scopes),taskname,task) - del taskname, task - continue - elif not only_scalars and token.token_value in [ - 'build', 'task', 'test', 'platform' ] \ - and peek.token_type=='varname': - taskname=peek.token_value - yell('%-7s %-7s %s\n'%( - 'PARSE',token.token_value,taskname)) - tokiter.next() # consume the task name - yell('%-7s %-7s %s\n'%( - 'INIT',token.token_value,taskname)) - if token.token_value=='task': - raise AssertionError('Should never make a Task') - task=Task(scopes,taskname) - elif token.token_value=='test': - task=Test(scopes,taskname,run_mode) - task.setlocal('TEST_NAME', - String(scopes,taskname,False)) - elif token.token_value=='build': - task=Build(scopes,taskname) - task.setlocal('BUILD_NAME', - String(scopes,taskname,False)) - elif token.token_value=='platform': - task=Platform(scopes,taskname) - task.setlocal('PLATFORM_NAME', - String(scopes,taskname,False)) - else: - raise AssertionError( - 'Unrecognized subscope type "%s".'%( - token.token_value,)) - task=self.parse_hash_define( - tokiter,scopes,task,parse_between, - allow_deps=token.token_value!='platform') - yell('%-7s %-7s %s\n'%('DEFINE',token.token_value, - taskname)) - define(self.con(peek,scopes),taskname,task) - del taskname, task - if parse_between: parse_between(tokiter) - continue - elif not only_scalars \ - and token.token_value=='hash' \ - and peek.token_type=='varname': - hashname=peek.token_value - tokiter.next() # consume the hash name - define(self.con(peek,scopes), - hashname,self.parse_hash_define( - tokiter,scopes,Scope(scopes),parse_between)) - del hashname - if parse_between: parse_between(tokiter) - continue - elif not only_scalars \ - and token.token_value=='embed' \ - and peek.token_type=='varname': - (varname,script)=self.parse_embed_script( - tokiter,scopes,parse_between) - define(self.con(peek,scopes),varname,script) - if parse_between: parse_between(tokiter) - continue - elif allow_null and ( - peek.token_value in ends or - parse_between and parse_between(tokiter)): - define(self.con(peek,scopes), - token.token_value,null_value) - if parse_between: parse_between(tokiter) - continue - elif token.token_type in ends: - return scopes[0] - elif token.token_type==end_of_line_type: - continue # ignore blank lines. - self.error(scope_name,token) - def parse_rvalue(self,tokiter,scopes,ends,parse_between=None, - only_scalars=False): - token=tokiter.next() - if token.token_type in [ 'qstring', 'dqstring', 'bracestring' ]: - ret=self.action_string(scopes,token) - if parse_between: parse_between(tokiter) - return ret - elif token.token_type == 'number': - ret=self.action_numeric(scopes,token) - if parse_between: parse_between(tokiter) - return ret - elif not only_scalars and token.token_type in '{': - subscope=Scope(scopes) - ret=self.parse_subscope( - tokiter,[subscope]+scopes,['}'], - self.parse_between_assignments, - allow_overwrite=True, - allow_resolve=True, - allow_run=False, - allow_null=False, - allow_use=True, - scope_name="hash") - if parse_between: parse_between(tokiter) - return ret - elif not only_scalars and token.token_type=='varname': - varname=token.token_value - peek=tokiter.peek() - if peek.token_type=='(': - # We are at the ( in varname(arguments... - tokiter.next() # consume ( - subscope=Scope(scopes) - scopesplus=[subscope]+scopes - self.parse_subscope(tokiter,scopesplus,[')'], - self.parse_between_arguments, - allow_overwrite=False, - allow_resolve=False, - allow_null=True, - scope_name='argument list') - peek=tokiter.peek() - if peek.token_type in ends or \ - parse_between and parse_between(tokiter): - # This is a function call varname(arg,arg,...) - return self.action_call(varname,peek,scopes,subscope) - elif peek.token_type in ends : - return self.action_resolve(token,scopes) - elif parse_between and parse_between(tokiter): - return self.action_resolve(token,scopes) - self.error('rvalue',token) - def action_autodetect(self,con,tokiter,scopes,taskname,task): - matches=task.detect(con) - if len(matches)==0: - raise Exception( - 'You are using an unknown platform. Fixme: need better ' - 'exception here.') - elif len(matches)>1: - raise Exception( - 'This machine can submit to multiple platforms: '+( - ' '.join([ - s.resolve('PLATFORM_NAME').string_context() - for s in matches - ]))) - return matches[0] - def action_dependency(self,task,scopes,dep): - task.add_dependency(dep) - def action_call(self,varname,token,scopes,parameters): - yell('%-7s %s in parameter scope %s\n'%( - 'CALL',repr(varname),repr(parameters))) - callme=scopes[0].resolve(varname) - yell('CALL APPLY PARAMETERS\n') - return callme.apply_parameters(parameters,self.con(token,scopes)) - def action_use(self,scopes,key_token,only_scalars=False): - assert(isinstance(key_token,Token)) - assert(isinstance(scopes,list)) - assert(len(scopes)>=2) - assert(isinstance(scopes[0],Scope)) - key=key_token.token_value - got=scopes[1].resolve(key) - found_non_scalars=scopes[0].use_from(got,only_scalars) - if only_scalars and found_non_scalars: - self.error('use',key_token,'found non-scalars when ' - 'using %s'%(key,)) - # for k,v in got.iterlocal(): - # if only_scalars and not isinstance(v,String): - # self.error('use',key_token,'found non-scalars when ' - # 'using %s'%(key,)) - # scopes[0].setlocal(k,v) - def action_operator(self,scopes,token): - assert(isinstance(token,Token)) - if token.token_value=='.copy.': - return Copy(scopes) - elif token.token_value=='.copydir.': - return CopyDir(scopes) - elif token.token_value=='.bitcmp.': - return BitCmp(scopes) - elif token.token_value=='.link.': - return Link(scopes) - elif token.token_value=='.atparse.': - return AtParse(scopes) - else: - self.error('operator name',token,'unknown operator '+ - token.token_value) - def action_numeric(self,scopes,token): - value=float(token.token_value) - return Numeric(value) - def action_string(self,scopes,token): - assert(isinstance(token,Token)) - if token.token_type=='qstring': - s=String(scopes,token.token_value,False) - elif token.token_type=='dqstring': - s=String(scopes,dqstring2bracestring(token.token_value),True) - elif token.token_type=='bracestring': - s=String(scopes,token.token_value,True) - else: - raise ValueError('Invalid token for a string: '+repr(token)) - yell('%-7s %s = %s\n'%('STRING',repr(token.token_value),repr(s))) - return s - def action_resolve(self,varname_token,scopes): - varname=varname_token.token_value - for scope in scopes: - try: - return scope.getlocal(varname) - except KeyError as ke: - pass - raise KeyError(varname) - def action_null_param(self,varname,scope): - if '%' in varname: - raise ValueError('%s: cannot have "%" in a parameter name.'%( - varname,)) - scope.check_define(varname,null_value) - def action_assign_var(self,toscope,tovar,fromvar,fromscopes, - allow_overwrite): - if fromscopes: - value=fromscopes[0].resolve(fromvar) - else: # Global scope assignment - value=toscope.resolve(fromvar) - self.action_assign(toscope,tovar,value,allow_overwrite) - def action_assign(self,scope,varname,value,allow_overwrite): - assert(isinstance(scope,Scope)) - assert(isinstance(varname,basestring)) - assert(isinstance(value,BaseObject)) - if '%' in varname: - raise ValueError('Cannot assign to %s; subscope definitions must be of syntax "var1 = { var2= { ...."'%( - varname,)) - yell('%-7s %s = %s IN %s\n'%( - 'ASSIGN', varname, repr(value),repr(scope) )) - if allow_overwrite: - scope.force_define(varname,value) - else: - scope.check_define(varname,value) - def action_run_obj(self,obj,con): - yell('%-7s %s\n'%( - 'RUN', repr(obj))) - self.add_run(obj,con) - def error(self,mode,token,reason=None): - if token is None: - raise Exception('Unexpected end of file.') - elif reason: - raise Exception('%s:%s: %s (%s token with value %s)'%( - token.filename,token.lineno,str(reason), - repr(token.token_type), - repr(elipses(str(token.token_value))))) - else: - raise Exception( - '%s:%s: unexpected %s in %s (token value %s)'%( - token.filename, token.lineno, repr(token.token_type), - str(mode), repr(elipses(str(token.token_value))))) - -######################################################################## - -class BashRunner(object): - def __init__(self): - super(BashRunner,self).__init__() - def make_runner(self,parser): - out=StringIO.StringIO() - out.write(r'''#! /usr/bin/env bash - -%s - -set -xe - -'''%(bash_functions,)) - - seen=False - for runme,con in parser.iterrun(): - seen=True - out.write(runme.bash_context(con)) - out.write("\n\n") - if not seen: - raise ValueError('ERROR: No "run" statments seen; nothing to do.\n'); - print out.getvalue() - out.close() - -######################################################################## - -tokenizer=Tokenizer() -parser=Parser() -global_scope=Scope([]) - -def process_file(fileobj,filename,first_line,scope): - parser.parse(TokenizeFile(tokenizer,fileobj,filename,first_line), - scope) - -modestr=sys.argv[1] -if modestr=='BASELINE': - run_mode=BASELINE -elif modestr=='EXECUTION': - run_mode=EXECUTION - -double_dash_seen=False -for arg in sys.argv[2:]: - if double_dash_seen: - with open(arg,'rt') as infile: - process_file(infile,arg,1,global_scope) - elif arg == '--': - double_dash_seen=True - elif arg == '-': - process_file(sys.stdin,'(**stdin**)',1,global_scope) - else: - with open(arg,'rt') as infile: - process_file(infile,arg,1,global_scope) - -yell('DONE DEFINING; RUN NOW\n') - -#BashRunner().make_runner(parser) -RocotoRunner().make_runner(parser) diff --git a/internal_tests/pytests/wrappers/pcp_combine/test_deprecated.conf b/internal_tests/pytests/wrappers/pcp_combine/test_deprecated.conf deleted file mode 100644 index 83d598e862..0000000000 --- a/internal_tests/pytests/wrappers/pcp_combine/test_deprecated.conf +++ /dev/null @@ -1,30 +0,0 @@ -[dir] -METPLUS_BASE = /d1/mccabe/METplus -OUTPUT_BASE = /d1/mccabe/test - -[config] -OBS_LEVEL = 1 -FCST_LEVEL = 6 - -OBS_DATA_INTERVAL = 1 -OBS_TIMES_PER_FILE = 4 -FCST_NATIVE_DATA_TYPE = NETCDF -FCST_6_FIELD_NAME = P06M_NONE - -OBS_1_FIELD_NAME = P01M_NONE - -OBS_PCP_COMBINE_INPUT_DIR = {METPLUS_BASE}/internal_tests/data/accum -OBS_PCP_COMBINE_OUTPUT_DIR = {METPLUS_BASE}/internal_tests/data/fakeout -OBS_NATIVE_DATA_TYPE = NETCDF - -#FCST_PCP_COMBINE_INPUT_DIR = {METPLUS_BASE}/internal_tests/data/accum -FCST_PCP_COMBINE_INPUT_DIR = {METPLUS_BASE}/internal_tests/data/fcst -FCST_PCP_COMBINE_OUTPUT_DIR = {METPLUS_BASE}/internal_tests/data/fakeout - -[filename_templates] -OBS_PCP_COMBINE_INPUT_TEMPLATE = {valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}.{level?fmt=%HH}h -OBS_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/outfile.{valid?fmt=%Y%m%d%H}_A{level?fmt=%HH}h -FCST_PCP_COMBINE_INPUT_TEMPLATE = {init?fmt=%Y%m%d}/file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc -FCST2_PCP_COMBINE_INPUT_TEMPLATE = file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc -#FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {init?fmt=%Y%m%d}/file.{init?fmt=%Y%m%d%H}f{lead?fmt=%HHH}.nc -FCST_PCP_COMBINE_OUTPUT_TEMPLATE = {valid?fmt=%Y%m%d}/file.{valid?fmt=%Y%m%d%H}_A{level?fmt=%HHH}.nc \ No newline at end of file diff --git a/internal_tests/replace_confs.py b/internal_tests/replace_confs.py deleted file mode 100755 index 3567edcd99..0000000000 --- a/internal_tests/replace_confs.py +++ /dev/null @@ -1,38 +0,0 @@ -# Script: replace_confs.py -# Author: George McCabe, November 2019 -# Description: Utility used to rename configuration variables in the METplus -# wrappers. Modify the script to set the list of directories to search and -# the dictionary containing the list of variables to replace. The script -# prints a list of commands to the screen. These commands can be run to -# find and replace all instances of the configuration variable listed. -# After running the commands, review 'git diff' to make sure that all of -# the changes are intended. For each change, documentation needs to be -# updated to list the new configuration variable name and deprecate the old -# name. - -import os - -# set this to the location of METplus you are using -top_level = '/d1/mccabe/METplus' - -# list of directories to find/replace -search_dirs = [ 'internal_tests/pytests', 'ush', 'parm' ] - -# dictionary of config items to be changed. key is old name, value is new name -change_dict = { - -"MODEL_DATA_DIR": "EXTRACT_TILES_GRID_INPUT_DIR", -"NLAT": "EXTRACT_TILES_NLAT", -"NLON": "EXTRACT_TILES_NLON", -"DLAT": "EXTRACT_TILES_DLAT", -"DLON": "EXTRACT_TILES_DLON", -"LON_ADJ": "EXTRACT_TILES_LON_ADJ", -"LAT_ADJ": "EXTRACT_TILES_LAT_ADJ", - -} - -for key, value in change_dict.items(): - for search_dir in search_dirs: - full_dir = os.path.join(top_level, search_dir) - cmd = f"egrep -lRZ '{key}' {full_dir}/. | xargs -0 -l sed -i -e 's/{key}/{value}/g'" - print(cmd) diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index 5430084d6c..44ee50ad8a 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -201,7 +201,7 @@ def print_doc_text(tool_name, input_dict): print("---------------------------------------------") print('\n==================================================\n') - print(f"In internal_tests/pytests/{tool_name}/" + print(f"In internal/tests/pytests/{tool_name}/" f"test_{tool_name}_wrapper.py" "\n\nAdd the following items to " "the tests to ensure the new items are set properly. Note: " diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index 24d536b77d..a8c337b8f4 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -1391,7 +1391,7 @@ def expand_int_string_to_list(int_string): def subset_list(full_list, subset_definition): """! Extract subset of items from full_list based on subset_definition - Used in internal_tests/use_cases/metplus_use_case_suite.py + Used in internal/tests/use_cases/metplus_use_case_suite.py @param full_list List of all use cases that were requested @param subset_definition Defines how to subset the full list. If None, diff --git a/produtil/externals b/produtil/externals index 623f98a1ac..5e2a63defe 100644 --- a/produtil/externals +++ b/produtil/externals @@ -82,7 +82,7 @@ commit: 9858fc849a8eba 2018 Jan 10 in the vlab git repository https://user.name@vlab.ncep.noaa.gov/code-review/a/EMC_produtil NOTE: EMC_produtil/ush/produtil moved to METplus/produtil -EMC_produtil/usy/*.py test files moved to METplus/internal_tests/produtil +EMC_produtil/usy/*.py test files moved to METplus/internal/tests/produtil ============================================