Generate JUnitXML from pytest

tests/scripts/setup-pytest-env.sh

@@ -27,4 +27,18 @@ fi
 set -u
 
 export TVM_PATH=`pwd`
-export PYTHONPATH=${TVM_PATH}/python
+export PYTHONPATH="${TVM_PATH}/python"
+
+export TVM_PYTEST_RESULT_DIR="${TVM_PATH}/build/pytest-results"
+mkdir -p "${TVM_PYTEST_RESULT_DIR}"
+
+function run_pytest() {
+    test_suite_name="$1"
+    shift
+    for ffi_type in ${TVM_PYTEST_FFI_TYPES:-ctypes cython}; do
+        TVM_FFI=${ffi_type} python3 -m pytest \
+           -o "junit_suite_name=${test_suite_name}-${ffi_type}" \
+           "--junit-xml=${TVM_PYTEST_RESULT_DIR}/${test_suite_name}-${ffi_type}.xml" \
+           "$@"
+    done
+}

tests/scripts/task_python_arm_compute_library.sh

@@ -22,9 +22,10 @@ source tests/scripts/setup-pytest-env.sh
 
 
 # Rebuild cython
+# TODO(u99127): Enable cython tests.
 
 find . -type f -path "*.pyc" | xargs rm -f
 make cython3
 
-TVM_FFI=ctypes python3 -m pytest tests/python/contrib/test_arm_compute_lib
-
+TVM_PYTEST_FFI_TYPES=ctypes run_pytest python-arm_compute_lib \
+                    tests/python/contrib/test_arm_compute_lib

tests/scripts/task_python_ethosn_tests.sh

@@ -22,9 +22,9 @@ source tests/scripts/setup-pytest-env.sh
 
 
 # Rebuild cython
+# TODO(u99127): Enable cython tests.
 
 find . -type f -path "*.pyc" | xargs rm -f
 make cython3
 
-TVM_FFI=ctypes python3 -m pytest tests/python/contrib/test_ethosn
-
+TVM_PYTEST_FFI_TYPES=ctypes run_pytest python-ethosn tests/python/contrib/test_ethosn

tests/scripts/task_python_frontend.sh

@@ -32,22 +32,22 @@ find . -type f -path "*.pyc" | xargs rm -f
 make cython3
 
 echo "Running relay MXNet frontend test..."
-python3 -m pytest tests/python/frontend/mxnet
+TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-mxnet tests/python/frontend/mxnet
 
 echo "Running relay ONNX frontend test..."
-python3 -m pytest tests/python/frontend/onnx
+TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-onnx tests/python/frontend/onnx
 
 echo "Running relay CoreML frontend test..."
-python3 -m pytest tests/python/frontend/coreml
+TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-coreml tests/python/frontend/coreml
 
 echo "Running relay Tensorflow frontend test..."
-python3 -m pytest tests/python/frontend/tensorflow
+TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-tensorflow tests/python/frontend/tensorflow
 
 echo "Running relay caffe2 frontend test..."
-python3 -m pytest tests/python/frontend/caffe2
+TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-caffe2 tests/python/frontend/caffe2
 
 echo "Running relay DarkNet frontend test..."
-python3 -m pytest tests/python/frontend/darknet
+TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-darknet tests/python/frontend/darknet
 
 echo "Running relay PyTorch frontend test..."
-python3 -m pytest tests/python/frontend/pytorch
+TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-pytorch tests/python/frontend/pytorch

tests/scripts/task_python_frontend_cpu.sh

@@ -33,10 +33,10 @@ find . -type f -path "*.pyc" | xargs rm -f
 make cython3
 
 echo "Running relay TFLite frontend test..."
-python3 -m pytest tests/python/frontend/tflite
+TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-tflite tests/python/frontend/tflite
 
 echo "Running relay Keras frontend test..."
-python3 -m pytest tests/python/frontend/keras
+TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-keras tests/python/frontend/keras
 
 echo "Running relay Caffe frontend test..."
-python3 -m pytest tests/python/frontend/caffe
+TVM_PYTHON_FFI_TYPES=cython run_pytest python-frontend-caffe tests/python/frontend/caffe

tests/scripts/task_python_integration.sh

@@ -27,6 +27,10 @@ export LD_LIBRARY_PATH="build:${LD_LIBRARY_PATH:-}"
 export TVM_BIND_THREADS=0
 export TVM_NUM_THREADS=2
 
+if [ -z "${TVM_INTEGRATION_TESTSUITE_NAME:-}" ]; then
+    TVM_INTEGRATION_TESTSUITE_NAME=python-integration
+fi
+
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
@@ -39,29 +43,28 @@ rm -rf lib
 make
 cd ../..
 
-TVM_FFI=cython python3 -m pytest apps/extension/tests
-TVM_FFI=ctypes python3 -m pytest apps/extension/tests
+run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-extensions apps/extension/tests
 
 # Test dso plugin
 cd apps/dso_plugin_module
 rm -rf lib
 make
 cd ../..
-TVM_FFI=cython python3 -m pytest apps/dso_plugin_module
-TVM_FFI=ctypes python3 -m pytest apps/dso_plugin_module
+run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-dso_plugin_module apps/dso_plugin_module
 
 # Do not enable TensorFlow op
 # TVM_FFI=cython sh prepare_and_test_tfop_module.sh
 # TVM_FFI=ctypes sh prepare_and_test_tfop_module.sh
 
-TVM_FFI=ctypes python3 -m pytest tests/python/integration
-TVM_FFI=ctypes python3 -m pytest tests/python/contrib
+TVM_PYTEST_FFI_TYPES=ctypes run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME} tests/python/integration
+TVM_PYTEST_FFI_TYPES=ctypes run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-contrib tests/python/contrib
 
-TVM_TEST_TARGETS="${TVM_RELAY_TEST_TARGETS:-llvm;cuda}" TVM_FFI=ctypes python3 -m pytest tests/python/relay
+TVM_TEST_TARGETS="${TVM_RELAY_TEST_TARGETS:-llvm;cuda}" \
+                TVM_PYTEST_FFI_TYPES=ctypes \
+                run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-relay tests/python/relay
 
 # Command line driver test
-TVM_FFI=ctypes python3 -m pytest tests/python/driver
+TVM_PYTEST_FFI_TYPES=ctypes run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-driver tests/python/driver
 
 # Do not enable OpenGL
-# TVM_FFI=cython python -m pytest tests/webgl
-# TVM_FFI=ctypes python3 -m pytest tests/webgl
+# run_pytest ${TVM_INTEGRATION_TESTSUITE_NAME}-webgl tests/webgl

tests/scripts/task_python_integration_gpuonly.sh

@@ -19,5 +19,6 @@
 export TVM_TEST_TARGETS="cuda;opencl;metal;rocm;vulkan;nvptx;opencl -device=mali,aocl_sw_emu"
 export PYTEST_ADDOPTS="-m gpu $PYTEST_ADDOPTS"
 export TVM_RELAY_TEST_TARGETS="cuda"
+export TVM_INTEGRATION_TESTSUITE_NAME=python-integration-gpu
 
 ./tests/scripts/task_python_integration.sh

tests/scripts/task_python_microtvm.sh

@@ -24,6 +24,5 @@ source tests/scripts/setup-pytest-env.sh
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
-TVM_FFI=ctypes python3 -m pytest tests/micro/qemu
 make cython3
-TVM_FFI=cython python3 -m pytest tests/micro/qemu
+run_pytest python-microtvm-qemu tests/micro/qemu

tests/scripts/task_python_nightly.sh

@@ -27,4 +27,4 @@ make cython3
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
-python3 -m pytest tests/python/topi/nightly
+TVM_PYTEST_FFI_TYPES=cython run_pytest python-topi-nightly tests/python/topi/nightly

tests/scripts/task_python_topi.sh

@@ -31,4 +31,4 @@ make cython3
 # cleanup pycache
 find . -type f -path "*.pyc" | xargs rm -f
 
-python3 -m pytest tests/python/topi/
+TVM_PYTHON_FFI_TYPES=cython run_pytest python-topi tests/python/topi/

tests/scripts/task_python_unittest.sh

@@ -25,7 +25,9 @@ source tests/scripts/setup-pytest-env.sh
 find . -type f -path "*.pyc" | xargs rm -f
 make cython3
 
-TVM_FFI=ctypes python3 -m pytest tests/python/all-platform-minimal-test
-TVM_FFI=cython python3 -m pytest tests/python/all-platform-minimal-test
-TVM_FFI=ctypes python3 -m pytest tests/python/unittest
-TVM_FFI=cython python3 -m pytest tests/python/unittest
+if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then
+    TVM_UNITTEST_TESTSUITE_NAME=python-unittest
+fi
+
+run_pytest ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test tests/python/all-platform-minimal-test
+run_pytest ${TVM_UNITTEST_TESTSUITE_NAME} tests/python/unittest

tests/scripts/task_python_unittest_gpuonly.sh

@@ -18,5 +18,6 @@
 
 export TVM_TEST_TARGETS="cuda;opencl;metal;rocm;vulkan;nvptx;opencl -device=mali,aocl_sw_emu"
 export PYTEST_ADDOPTS="-m gpu $PYTEST_ADDOPTS"
+export TVM_UNITTEST_TESTSUITE_NAME=python-unittest-gpu
 
 ./tests/scripts/task_python_unittest.sh

tests/scripts/task_python_vta_fsim.sh

@@ -40,8 +40,10 @@ cp ${VTA_HW_PATH}/config/fsim_sample.json ${VTA_HW_PATH}/config/vta_config.json
 
 # Run unit tests in functional/fast simulator
 echo "Running unittest in fsim..."
-python3 -m pytest ${TVM_PATH}/vta/tests/python/unittest
+TVM_PYTEST_FFI_TYPES=cython run_pytest python-vta-fsim-unittest \
+                    ${TVM_PATH}/vta/tests/python/unittest
 
 # Run unit tests in functional/fast simulator
 echo "Running integration test in fsim..."
-python3 -m pytest ${TVM_PATH}/vta/tests/python/integration
+TVM_PYTEST_FFI_TYPES=cython run_pytest python-vta-fsim-integration \
+                    ${TVM_PATH}/vta/tests/python/integration

tests/scripts/task_python_vta_tsim.sh

@@ -55,11 +55,13 @@ make -C ${VTA_HW_PATH}/hardware/chisel USE_THREADS=0 lib
 
 # Run unit tests in cycle accurate simulator
 echo "Running unittest in tsim..."
-python3 -m pytest ${TVM_PATH}/vta/tests/python/unittest
+TVM_PYTEST_FFI_TYPES=cython run_pytest python-vta-tsim-unittest \
+                    ${TVM_PATH}/vta/tests/python/unittest
 
 # Run unit tests in cycle accurate simulator
 echo "Running integration test in tsim..."
-python3 -m pytest ${TVM_PATH}/vta/tests/python/integration
+TVM_PYTEST_FFI_TYPES=cython run_pytest python-vta-tsim-integration \
+                    ${TVM_PATH}/vta/tests/python/integration
 
 # Reset default fsim simulation
 cp ${VTA_HW_PATH}/config/fsim_sample.json ${VTA_HW_PATH}/config/vta_config.json