[microTVM] Refactor pytest fixtures (apache#12207)

* Refactor micro test fixtures

* fix error

* fix scope

* address @guberti comments

* fix help message

* rename tvm_debug and added .gitignore

* fix bug

* fix bug

python/tvm/micro/testing/evaluation.py

@@ -28,6 +28,7 @@
 import tempfile
 
 import tvm
+from tvm.relay.op.contrib import cmsisnn
 
 
 def tune_model(
@@ -99,6 +100,8 @@ def create_aot_session(
         if use_cmsis_nn:
             config["relay.ext.cmsisnn.options"] = {"mcpu": target.mcpu}
         stack.enter_context(tvm.transform.PassContext(opt_level=3, config=config))
+        if use_cmsis_nn:
+            mod = cmsisnn.partition_for_cmsisnn(mod, params, mcpu=target.mcpu)
         if tune_logs is not None:
             stack.enter_context(tvm.autotvm.apply_history_best(tune_logs))
 
@@ -153,4 +156,4 @@ def evaluate_model_accuracy(session, aot_executor, input_data, true_labels, runs
     num_correct = sum(u == v for u, v in zip(true_labels, predicted_labels))
     average_time = sum(aot_runtimes) / len(aot_runtimes)
     accuracy = num_correct / len(predicted_labels)
-    return average_time, accuracy
+    return average_time, accuracy, predicted_labels

python/tvm/micro/testing/pytest_plugin.py

@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# pylint: disable=invalid-name,redefined-outer-name
+""" microTVM testing fixtures used to deduce testing argument
+    values from testing parameters """
+
+import pathlib
+import os
+import datetime
+import pytest
+
+from tvm.contrib.utils import tempdir
+
+from .utils import get_supported_boards
+
+
+def pytest_addoption(parser):
+    """Adds more pytest arguments"""
+    parser.addoption(
+        "--board",
+        required=True,
+        choices=list(get_supported_boards("zephyr").keys())
+        + list(get_supported_boards("arduino").keys()),
+        help=(
+            "microTVM boards for tests. Board refers to instances"
+            "of microcontrollers/emulators defined in a platform."
+        ),
+    )
+    parser.addoption(
+        "--test-build-only",
+        action="store_true",
+        help="Only run tests that don't require physical hardware.",
+    )
+    parser.addoption(
+        "--microtvm-debug",
+        action="store_true",
+        default=False,
+        help=(
+            "If set true, it will keep the project directory for debugging."
+            "Also, it will enable debug level logging in project generation."
+        ),
+    )
+
+
+@pytest.fixture(scope="session")
+def board(request):
+    return request.config.getoption("--board")
+
+
+@pytest.fixture(scope="session")
+def microtvm_debug(request):
+    return request.config.getoption("--microtvm-debug")
+
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--test-build-only"):
+        skip_hardware_tests = pytest.mark.skip(reason="--test-build-only was passed")
+        for item in items:
+            if "requires_hardware" in item.keywords:
+                item.add_marker(skip_hardware_tests)
+
+
+@pytest.fixture
+def workspace_dir(request, board, microtvm_debug):
+    """Creates workspace directory for each test."""
+    parent_dir = pathlib.Path(os.path.dirname(request.module.__file__))
+    board_workspace = (
+        parent_dir / f"workspace_{board}" / datetime.datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
+    )
+    board_workspace_base = str(board_workspace)
+    number = 1
+    while board_workspace.exists():
+        board_workspace = pathlib.Path(board_workspace_base + f"-{number}")
+        number += 1
+
+    if not os.path.exists(board_workspace.parent):
+        os.makedirs(board_workspace.parent)
+
+    keep_for_debug = microtvm_debug if microtvm_debug else None
+    test_temp_dir = tempdir(custom_path=board_workspace, keep_for_debug=keep_for_debug)
+    return test_temp_dir
+
+
+@pytest.fixture(autouse=True)
+def skip_by_board(request, board):
+    """Skip test if board is in the list."""
+    if request.node.get_closest_marker("skip_boards"):
+        if board in request.node.get_closest_marker("skip_boards").args[0]:
+            pytest.skip("skipped on this board: {}".format(board))
+
+
+def pytest_configure(config):
+    config.addinivalue_line(
+        "markers",
+        "skip_boards(board): skip test for the given board",
+    )

tests/micro/.gitignore

@@ -0,0 +1,2 @@
+# test workspaces
+*/workspace_*/

tests/micro/arduino/conftest.py

@@ -15,35 +15,19 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import pytest
+pytest_plugins = [
+    "tvm.micro.testing.pytest_plugin",
+]
 
-from test_utils import ARDUINO_BOARDS
+import pytest
 
 
 def pytest_addoption(parser):
-    parser.addoption(
-        "--arduino-board",
-        nargs="+",
-        required=True,
-        choices=ARDUINO_BOARDS.keys(),
-        help="Arduino board for tests.",
-    )
     parser.addoption(
         "--arduino-cli-cmd",
         default="arduino-cli",
         help="Path to `arduino-cli` command for flashing device.",
     )
-    parser.addoption(
-        "--test-build-only",
-        action="store_true",
-        help="Only run tests that don't require physical hardware.",
-    )
-    parser.addoption(
-        "--tvm-debug",
-        action="store_true",
-        default=False,
-        help="If given, enable a debug session while the test is running. Before running the test, in a separate shell, you should run: <python -m tvm.exec.microtvm_debug_shell>",
-    )
 
 
 def pytest_configure(config):
@@ -52,27 +36,6 @@ def pytest_configure(config):
     )
 
 
-def pytest_collection_modifyitems(config, items):
-    if config.getoption("--test-build-only"):
-        skip_hardware_tests = pytest.mark.skip(reason="--test-build-only was passed")
-        for item in items:
-            if "requires_hardware" in item.keywords:
-                item.add_marker(skip_hardware_tests)
-
-
-# We might do project generation differently for different boards in the future
-# (to take advantage of multiple cores / external memory / etc.), so all tests
-# are parameterized by board
-def pytest_generate_tests(metafunc):
-    board = metafunc.config.getoption("arduino_board")
-    metafunc.parametrize("board", board, scope="session")
-
-
 @pytest.fixture(scope="session")
 def arduino_cli_cmd(request):
     return request.config.getoption("--arduino-cli-cmd")
-
-
-@pytest.fixture(scope="session")
-def tvm_debug(request):
-    return request.config.getoption("--tvm-debug")

tests/micro/arduino/test_arduino_error_detection.py

@@ -23,15 +23,10 @@
 import test_utils
 import tvm.testing
 
-# A new project and workspace dir is created for EVERY test
-@pytest.fixture
-def workspace_dir(request, board):
-    return test_utils.make_workspace_dir("arduino_error_detection", board)
-
 
 @pytest.fixture
-def project(board, arduino_cli_cmd, tvm_debug, workspace_dir):
-    return test_utils.make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir)
+def project(board, arduino_cli_cmd, microtvm_debug, workspace_dir):
+    return test_utils.make_kws_project(board, arduino_cli_cmd, microtvm_debug, workspace_dir)
 
 
 def test_blank_project_compiles(workspace_dir, project):

tests/micro/arduino/test_arduino_rpc_server.py

@@ -37,11 +37,6 @@
 
 import test_utils
 
-# # A new project and workspace dir is created for EVERY test
-@pytest.fixture
-def workspace_dir(board):
-    return test_utils.make_workspace_dir("arduino_rpc_server", board)
-
 
 def _make_session(model, arduino_board, arduino_cli_cmd, workspace_dir, mod, build_config):
     project = tvm.micro.generate_project(
@@ -84,11 +79,11 @@ def _make_add_sess(model, arduino_board, arduino_cli_cmd, workspace_dir, build_c
 # The same test code can be executed on both the QEMU simulation and on real hardware.
 @tvm.testing.requires_micro
 @pytest.mark.requires_hardware
-def test_compile_runtime(board, arduino_cli_cmd, tvm_debug, workspace_dir):
+def test_compile_runtime(board, arduino_cli_cmd, microtvm_debug, workspace_dir):
     """Test compiling the on-device runtime."""
 
     model = test_utils.ARDUINO_BOARDS[board]
-    build_config = {"debug": tvm_debug}
+    build_config = {"debug": microtvm_debug}
 
     # NOTE: run test in a nested function so cPython will delete arrays before closing the session.
     def test_basic_add(sess):
@@ -109,11 +104,11 @@ def test_basic_add(sess):
 
 @tvm.testing.requires_micro
 @pytest.mark.requires_hardware
-def test_platform_timer(board, arduino_cli_cmd, tvm_debug, workspace_dir):
+def test_platform_timer(board, arduino_cli_cmd, microtvm_debug, workspace_dir):
     """Test compiling the on-device runtime."""
 
     model = test_utils.ARDUINO_BOARDS[board]
-    build_config = {"debug": tvm_debug}
+    build_config = {"debug": microtvm_debug}
 
     # NOTE: run test in a nested function so cPython will delete arrays before closing the session.
     def test_basic_add(sess):
@@ -139,10 +134,10 @@ def test_basic_add(sess):
 
 @tvm.testing.requires_micro
 @pytest.mark.requires_hardware
-def test_relay(board, arduino_cli_cmd, tvm_debug, workspace_dir):
+def test_relay(board, arduino_cli_cmd, microtvm_debug, workspace_dir):
     """Testing a simple relay graph"""
     model = test_utils.ARDUINO_BOARDS[board]
-    build_config = {"debug": tvm_debug}
+    build_config = {"debug": microtvm_debug}
 
     shape = (10,)
     dtype = "int8"
@@ -172,10 +167,10 @@ def test_relay(board, arduino_cli_cmd, tvm_debug, workspace_dir):
 
 @tvm.testing.requires_micro
 @pytest.mark.requires_hardware
-def test_onnx(board, arduino_cli_cmd, tvm_debug, workspace_dir):
+def test_onnx(board, arduino_cli_cmd, microtvm_debug, workspace_dir):
     """Testing a simple ONNX model."""
     model = test_utils.ARDUINO_BOARDS[board]
-    build_config = {"debug": tvm_debug}
+    build_config = {"debug": microtvm_debug}
 
     # Load test images.
     this_dir = pathlib.Path(__file__).parent
@@ -263,10 +258,10 @@ def check_result(
 
 @tvm.testing.requires_micro
 @pytest.mark.requires_hardware
-def test_byoc_microtvm(board, arduino_cli_cmd, tvm_debug, workspace_dir):
+def test_byoc_microtvm(board, arduino_cli_cmd, microtvm_debug, workspace_dir):
     """This is a simple test case to check BYOC capabilities of microTVM"""
     model = test_utils.ARDUINO_BOARDS[board]
-    build_config = {"debug": tvm_debug}
+    build_config = {"debug": microtvm_debug}
 
     x = relay.var("x", shape=(10, 10))
     w0 = relay.var("w0", shape=(10, 10))
@@ -347,10 +342,10 @@ def _make_add_sess_with_shape(
 )
 @tvm.testing.requires_micro
 @pytest.mark.requires_hardware
-def test_rpc_large_array(board, arduino_cli_cmd, tvm_debug, workspace_dir, shape):
+def test_rpc_large_array(board, arduino_cli_cmd, microtvm_debug, workspace_dir, shape):
     """Test large RPC array transfer."""
     model = test_utils.ARDUINO_BOARDS[board]
-    build_config = {"debug": tvm_debug}
+    build_config = {"debug": microtvm_debug}
 
     # NOTE: run test in a nested function so cPython will delete arrays before closing the session.
     def test_tensors(sess):

tests/micro/arduino/test_arduino_workflow.py

@@ -36,23 +36,24 @@
 6. Use serial connection to ensure model behaves correctly
 """
 
-
 # Since these tests are sequential, we'll use the same project/workspace
 # directory for all tests in this file
 @pytest.fixture(scope="module")
-def workspace_dir(request, board):
+def workflow_workspace_dir(request, board):
     return test_utils.make_workspace_dir("arduino_workflow", board)
 
 
 @pytest.fixture(scope="module")
-def project_dir(workspace_dir):
-    return workspace_dir / "project"
+def project_dir(workflow_workspace_dir):
+    return workflow_workspace_dir / "project"
 
 
 # We MUST pass workspace_dir, not project_dir, or the workspace will be dereferenced too soon
 @pytest.fixture(scope="module")
-def project(board, arduino_cli_cmd, tvm_debug, workspace_dir):
-    return test_utils.make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir)
+def project(board, arduino_cli_cmd, microtvm_debug, workflow_workspace_dir):
+    return test_utils.make_kws_project(
+        board, arduino_cli_cmd, microtvm_debug, workflow_workspace_dir
+    )
 
 
 def _get_directory_elements(directory):

tests/micro/arduino/test_utils.py

@@ -61,10 +61,10 @@ def make_workspace_dir(test_name, board):
     return t
 
 
-def make_kws_project(board, arduino_cli_cmd, tvm_debug, workspace_dir):
+def make_kws_project(board, arduino_cli_cmd, microtvm_debug, workspace_dir):
     this_dir = pathlib.Path(__file__).parent
     model = ARDUINO_BOARDS[board]
-    build_config = {"debug": tvm_debug}
+    build_config = {"debug": microtvm_debug}
 
     mod, params = fetch_model_from_url(
         url="https://github.com/tensorflow/tflite-micro/raw/main/tensorflow/lite/micro/examples/micro_speech/micro_speech.tflite",