From 37c3c5d78849ea39b30dcf0da10e327201590d30 Mon Sep 17 00:00:00 2001 From: Andrew Reusch Date: Thu, 4 Feb 2021 13:54:49 -0800 Subject: [PATCH 1/2] Generate JUnitXML from pytest. --- tests/scripts/setup-pytest-env.sh | 16 ++++++++++++- .../task_python_arm_compute_library.sh | 5 ++-- tests/scripts/task_python_ethosn_tests.sh | 4 ++-- tests/scripts/task_python_frontend.sh | 14 +++++------ tests/scripts/task_python_frontend_cpu.sh | 6 ++--- tests/scripts/task_python_integration.sh | 23 +++++++++++-------- .../task_python_integration_gpuonly.sh | 1 + tests/scripts/task_python_microtvm.sh | 3 +-- tests/scripts/task_python_nightly.sh | 2 +- tests/scripts/task_python_topi.sh | 2 +- tests/scripts/task_python_unittest.sh | 10 ++++---- tests/scripts/task_python_unittest_gpuonly.sh | 1 + tests/scripts/task_python_vta_fsim.sh | 6 +++-- tests/scripts/task_python_vta_tsim.sh | 6 +++-- 14 files changed, 62 insertions(+), 37 deletions(-) diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh index 475ce1ce1c53..d7e9b4db6d42 100755 --- a/tests/scripts/setup-pytest-env.sh +++ b/tests/scripts/setup-pytest-env.sh @@ -27,4 +27,18 @@ fi set -u export TVM_PATH=`pwd` -export PYTHONPATH=${TVM_PATH}/python +export PYTHONPATH="${TVM_PATH}/python" + +export TVM_PYTEST_RESULT_DIR="${TVM_PATH}/build/pytest-results" +mkdir -p "${TVM_PYTEST_RESULT_DIR}" + +function run_pytest() { + test_suite_name="$1" + shift + for ffi_type in ${TVM_PYTEST_FFI_TYPES:-ctypes cython}; do + TVM_FFI=${ffi_type} python3 -m pytest \ + -o "junit_suite_name=${test_suite_name}-ctypes" \ + "--junit-xml=${TVM_PYTEST_RESULT_DIR}/${test_suite_name}-${ffi_type}.xml" \ + "$@" + done +} diff --git a/tests/scripts/task_python_arm_compute_library.sh b/tests/scripts/task_python_arm_compute_library.sh index e36d042676d6..4c1992b58692 100755 --- a/tests/scripts/task_python_arm_compute_library.sh +++ b/tests/scripts/task_python_arm_compute_library.sh @@ -22,9 +22,10 @@ source tests/scripts/setup-pytest-env.sh # Rebuild cython +# TODO(u99127): Enable cython tests. find . -type f -path "*.pyc" | xargs rm -f make cython3 -TVM_FFI=ctypes python3 -m pytest tests/python/contrib/test_arm_compute_lib - +TVM_PYTEST_FFI_TYPES=ctypes run_pytest python-arm_compute_lib \ + tests/python/contrib/test_arm_compute_lib diff --git a/tests/scripts/task_python_ethosn_tests.sh b/tests/scripts/task_python_ethosn_tests.sh index 36a3d0919650..472ca38149a1 100755 --- a/tests/scripts/task_python_ethosn_tests.sh +++ b/tests/scripts/task_python_ethosn_tests.sh @@ -22,9 +22,9 @@ source tests/scripts/setup-pytest-env.sh # Rebuild cython +# TODO(u99127): Enable cython tests. find . -type f -path "*.pyc" | xargs rm -f make cython3 -TVM_FFI=ctypes python3 -m pytest tests/python/contrib/test_ethosn - +TVM_PYTEST_FFI_TYPES=ctypes run_pytest python-ethosn tests/python/contrib/test_ethosn diff --git a/tests/scripts/task_python_frontend.sh b/tests/scripts/task_python_frontend.sh index 3c5839bc7e1c..6b1d8e5038fb 100755 --- a/tests/scripts/task_python_frontend.sh +++ b/tests/scripts/task_python_frontend.sh @@ -32,22 +32,22 @@ find . -type f -path "*.pyc" | xargs rm -f make cython3 echo "Running relay MXNet frontend test..." -python3 -m pytest tests/python/frontend/mxnet +TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-mxnet tests/python/frontend/mxnet echo "Running relay ONNX frontend test..." -python3 -m pytest tests/python/frontend/onnx +TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-onnx tests/python/frontend/onnx echo "Running relay CoreML frontend test..." -python3 -m pytest tests/python/frontend/coreml +TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-coreml tests/python/frontend/coreml echo "Running relay Tensorflow frontend test..." -python3 -m pytest tests/python/frontend/tensorflow +TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-tensorflow tests/python/frontend/tensorflow echo "Running relay caffe2 frontend test..." -python3 -m pytest tests/python/frontend/caffe2 +TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-caffe2 tests/python/frontend/caffe2 echo "Running relay DarkNet frontend test..." -python3 -m pytest tests/python/frontend/darknet +TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-darknet tests/python/frontend/darknet echo "Running relay PyTorch frontend test..." -python3 -m pytest tests/python/frontend/pytorch +TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-pytorch tests/python/frontend/pytorch diff --git a/tests/scripts/task_python_frontend_cpu.sh b/tests/scripts/task_python_frontend_cpu.sh index 6dfcabc2cd37..a5cd3ba8ef09 100755 --- a/tests/scripts/task_python_frontend_cpu.sh +++ b/tests/scripts/task_python_frontend_cpu.sh @@ -33,10 +33,10 @@ find . -type f -path "*.pyc" | xargs rm -f make cython3 echo "Running relay TFLite frontend test..." -python3 -m pytest tests/python/frontend/tflite +TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-tflite tests/python/frontend/tflite echo "Running relay Keras frontend test..." -python3 -m pytest tests/python/frontend/keras +TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-keras tests/python/frontend/keras echo "Running relay Caffe frontend test..." -python3 -m pytest tests/python/frontend/caffe +TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-caffe tests/python/frontend/caffe diff --git a/tests/scripts/task_python_integration.sh b/tests/scripts/task_python_integration.sh index ef86d6917424..dc96097fb115 100755 --- a/tests/scripts/task_python_integration.sh +++ b/tests/scripts/task_python_integration.sh @@ -27,6 +27,10 @@ export LD_LIBRARY_PATH="build:${LD_LIBRARY_PATH:-}" export TVM_BIND_THREADS=0 export TVM_NUM_THREADS=2 +if [ -z "${TVM_INTEGRATION_TESTSUITE_NAME:-}" ]; then + TVM_INTEGRATION_TESTSUITE_NAME=python-integration +fi + # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f @@ -39,29 +43,28 @@ rm -rf lib make cd ../.. -TVM_FFI=cython python3 -m pytest apps/extension/tests -TVM_FFI=ctypes python3 -m pytest apps/extension/tests +run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-extensions apps/extension/tests # Test dso plugin cd apps/dso_plugin_module rm -rf lib make cd ../.. -TVM_FFI=cython python3 -m pytest apps/dso_plugin_module -TVM_FFI=ctypes python3 -m pytest apps/dso_plugin_module +run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-dso_plugin_module apps/dso_plugin_module # Do not enable TensorFlow op # TVM_FFI=cython sh prepare_and_test_tfop_module.sh # TVM_FFI=ctypes sh prepare_and_test_tfop_module.sh -TVM_FFI=ctypes python3 -m pytest tests/python/integration -TVM_FFI=ctypes python3 -m pytest tests/python/contrib +TVM_PYTEST_FFI_TYPES=ctypes run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME} tests/python/integration +TVM_PYTEST_FFI_TYPES=ctypes run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-contrib tests/python/contrib -TVM_TEST_TARGETS="${TVM_RELAY_TEST_TARGETS:-llvm;cuda}" TVM_FFI=ctypes python3 -m pytest tests/python/relay +TVM_TEST_TARGETS="${TVM_RELAY_TEST_TARGETS:-llvm;cuda}" \ + TVM_PYTEST_FFI_TYPES=ctypes \ + run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-relay tests/python/relay # Command line driver test -TVM_FFI=ctypes python3 -m pytest tests/python/driver +TVM_PYTEST_FFI_TYPES=ctypes run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-driver tests/python/driver # Do not enable OpenGL -# TVM_FFI=cython python -m pytest tests/webgl -# TVM_FFI=ctypes python3 -m pytest tests/webgl +# run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-webgl tests/webgl diff --git a/tests/scripts/task_python_integration_gpuonly.sh b/tests/scripts/task_python_integration_gpuonly.sh index c2a9e0c15abe..ac09cb5a14a3 100755 --- a/tests/scripts/task_python_integration_gpuonly.sh +++ b/tests/scripts/task_python_integration_gpuonly.sh @@ -19,5 +19,6 @@ export TVM_TEST_TARGETS="cuda;opencl;metal;rocm;vulkan;nvptx;opencl -device=mali,aocl_sw_emu" export PYTEST_ADDOPTS="-m gpu $PYTEST_ADDOPTS" export TVM_RELAY_TEST_TARGETS="cuda" +export TVM_INTEGRATION_TESTSUITE_NAME=python-integration-gpu ./tests/scripts/task_python_integration.sh diff --git a/tests/scripts/task_python_microtvm.sh b/tests/scripts/task_python_microtvm.sh index 7fb8d471a53a..ddedff37c6c2 100755 --- a/tests/scripts/task_python_microtvm.sh +++ b/tests/scripts/task_python_microtvm.sh @@ -24,6 +24,5 @@ source tests/scripts/setup-pytest-env.sh # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f -TVM_FFI=ctypes python3 -m pytest tests/micro/qemu make cython3 -TVM_FFI=cython python3 -m pytest tests/micro/qemu +run_pytest python-microtvm-qemu tests/micro/qemu diff --git a/tests/scripts/task_python_nightly.sh b/tests/scripts/task_python_nightly.sh index 36a620541997..bff0650b0bed 100755 --- a/tests/scripts/task_python_nightly.sh +++ b/tests/scripts/task_python_nightly.sh @@ -27,4 +27,4 @@ make cython3 # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f -python3 -m pytest tests/python/topi/nightly +TVM_PYTEST_FFI_TYPES=cython run_pytest python-topi-nightly tests/python/topi/nightly diff --git a/tests/scripts/task_python_topi.sh b/tests/scripts/task_python_topi.sh index 3bc3caf825cf..e5eb6f28276a 100755 --- a/tests/scripts/task_python_topi.sh +++ b/tests/scripts/task_python_topi.sh @@ -31,4 +31,4 @@ make cython3 # cleanup pycache find . -type f -path "*.pyc" | xargs rm -f -python3 -m pytest tests/python/topi/ +TVM_PYTHON_FFI_TYPES=cython run_pytest python-topi tests/python/topi/ diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh index 0aaf9fc86664..b63d79a99562 100755 --- a/tests/scripts/task_python_unittest.sh +++ b/tests/scripts/task_python_unittest.sh @@ -25,7 +25,9 @@ source tests/scripts/setup-pytest-env.sh find . -type f -path "*.pyc" | xargs rm -f make cython3 -TVM_FFI=ctypes python3 -m pytest tests/python/all-platform-minimal-test -TVM_FFI=cython python3 -m pytest tests/python/all-platform-minimal-test -TVM_FFI=ctypes python3 -m pytest tests/python/unittest -TVM_FFI=cython python3 -m pytest tests/python/unittest +if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then + TVM_UNITTEST_TESTSUITE_NAME=python-unittest +fi + +run_pytest ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test tests/python/all-platform-minimal-test +run_pytest ${TVM_UNITTEST_TESTSUITE_NAME} tests/python/unittest diff --git a/tests/scripts/task_python_unittest_gpuonly.sh b/tests/scripts/task_python_unittest_gpuonly.sh index 56722b16a364..22f79bc70ec9 100755 --- a/tests/scripts/task_python_unittest_gpuonly.sh +++ b/tests/scripts/task_python_unittest_gpuonly.sh @@ -18,5 +18,6 @@ export TVM_TEST_TARGETS="cuda;opencl;metal;rocm;vulkan;nvptx;opencl -device=mali,aocl_sw_emu" export PYTEST_ADDOPTS="-m gpu $PYTEST_ADDOPTS" +export TVM_UNITTEST_TESTSUITE_NAME=python-unittest-gpu ./tests/scripts/task_python_unittest.sh diff --git a/tests/scripts/task_python_vta_fsim.sh b/tests/scripts/task_python_vta_fsim.sh index 8080bbe756c7..74d14db95d30 100755 --- a/tests/scripts/task_python_vta_fsim.sh +++ b/tests/scripts/task_python_vta_fsim.sh @@ -40,8 +40,10 @@ cp ${VTA_HW_PATH}/config/fsim_sample.json ${VTA_HW_PATH}/config/vta_config.json # Run unit tests in functional/fast simulator echo "Running unittest in fsim..." -python3 -m pytest ${TVM_PATH}/vta/tests/python/unittest +TVM_PYTEST_FFI_TYPES=cython run_pytest python-vta-fsim-unittest \ + ${TVM_PATH}/vta/tests/python/unittest # Run unit tests in functional/fast simulator echo "Running integration test in fsim..." -python3 -m pytest ${TVM_PATH}/vta/tests/python/integration +TVM_PYTEST_FFI_TYPES=cython run_pytest python-vta-fsim-integration \ + ${TVM_PATH}/vta/tests/python/integration diff --git a/tests/scripts/task_python_vta_tsim.sh b/tests/scripts/task_python_vta_tsim.sh index c87d5483b8a5..4a5c9d7da877 100755 --- a/tests/scripts/task_python_vta_tsim.sh +++ b/tests/scripts/task_python_vta_tsim.sh @@ -55,11 +55,13 @@ make -C ${VTA_HW_PATH}/hardware/chisel USE_THREADS=0 lib # Run unit tests in cycle accurate simulator echo "Running unittest in tsim..." -python3 -m pytest ${TVM_PATH}/vta/tests/python/unittest +TVM_PYTEST_FFI_TYPES=cython run_pytest python-vta-tsim-unittest \ + ${TVM_PATH}/vta/tests/python/unittest # Run unit tests in cycle accurate simulator echo "Running integration test in tsim..." -python3 -m pytest ${TVM_PATH}/vta/tests/python/integration +TVM_PYTEST_FFI_TYPES=cython run_pytest python-vta-tsim-integration \ + ${TVM_PATH}/vta/tests/python/integration # Reset default fsim simulation cp ${VTA_HW_PATH}/config/fsim_sample.json ${VTA_HW_PATH}/config/vta_config.json From 72a58aa027407bf74082b02e964096319c062a33 Mon Sep 17 00:00:00 2001 From: Andrew Reusch Date: Fri, 5 Feb 2021 12:57:52 -0800 Subject: [PATCH 2/2] address tkonolige comments --- tests/scripts/setup-pytest-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh index d7e9b4db6d42..b77d3f37cd3e 100755 --- a/tests/scripts/setup-pytest-env.sh +++ b/tests/scripts/setup-pytest-env.sh @@ -37,7 +37,7 @@ function run_pytest() { shift for ffi_type in ${TVM_PYTEST_FFI_TYPES:-ctypes cython}; do TVM_FFI=${ffi_type} python3 -m pytest \ - -o "junit_suite_name=${test_suite_name}-ctypes" \ + -o "junit_suite_name=${test_suite_name}-${ffi_type}" \ "--junit-xml=${TVM_PYTEST_RESULT_DIR}/${test_suite_name}-${ffi_type}.xml" \ "$@" done