From d8a1b0e7f19ba922e673afced026a3426d1bc795 Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Fri, 17 Sep 2021 15:26:33 -0700 Subject: [PATCH] address comments --- .../template_project/microtvm_api_server.py | 30 +++---- python/tvm/micro/project.py | 17 +--- tests/micro/zephyr/conftest.py | 57 +------------ tests/micro/zephyr/test_utils.py | 62 ++++++++++++++ tests/micro/zephyr/test_zephyr.py | 81 ++++++++----------- tests/micro/zephyr/test_zephyr_aot.py | 15 +--- 6 files changed, 121 insertions(+), 141 deletions(-) create mode 100644 tests/micro/zephyr/test_utils.py diff --git a/apps/microtvm/zephyr/template_project/microtvm_api_server.py b/apps/microtvm/zephyr/template_project/microtvm_api_server.py index db45cab8b622..ed275e610912 100644 --- a/apps/microtvm/zephyr/template_project/microtvm_api_server.py +++ b/apps/microtvm/zephyr/template_project/microtvm_api_server.py @@ -58,15 +58,16 @@ IS_TEMPLATE = not (API_SERVER_DIR / MODEL_LIBRARY_FORMAT_RELPATH).exists() + +BOARDS = API_SERVER_DIR / "boards.json" + # Data structure to hold the information microtvm_api_server.py needs # to communicate with each of these boards. -BOARD_PROPERTIES = None -BOARDS_FILE_NAME = "boards.json" - -with open( - BOARDS_FILE_NAME, -) as board_f: - BOARD_PROPERTIES = json.load(board_f) +try: + with open(BOARDS) as boards: + BOARD_PROPERTIES = json.load(boards) +except FileNotFoundError: + raise FileNotFoundError(f"Board file {{{BOARDS}}} does not exist.") def check_call(cmd_args, *args, **kwargs): @@ -260,12 +261,7 @@ def _get_nrf_device_args(options): help="Name of the Zephyr board to build for.", ), server.ProjectOption( - "zephyr_model", - choices=[board["model"] for _, board in BOARD_PROPERTIES.items()], - help="Name of the model for each Zephyr board.", - ), - server.ProjectOption( - "main_stack_size", + "config_main_stack_size", help="Sets CONFIG_MAIN_STACK_SIZE for Zephyr board.", ), ] @@ -326,8 +322,8 @@ def _create_prj_conf(self, project_dir, options): f.write("# For models with floating point.\n" "CONFIG_FPU=y\n" "\n") # Set main stack size, if needed. - if options["main_stack_size"] is not None: - f.write(f"CONFIG_MAIN_STACK_SIZE={options['main_stack_size']}\n") + if options.get("config_main_stack_size") is not None: + f.write(f"CONFIG_MAIN_STACK_SIZE={options['config_main_stack_size']}\n") f.write("# For random number generation.\n" "CONFIG_TEST_RANDOM_GENERATOR=y\n") @@ -355,9 +351,7 @@ def generate_project(self, model_library_format_path, standalone_crt_dir, projec shutil.copy2(__file__, project_dir / os.path.basename(__file__)) # Copy boards.json file to generated project. - shutil.copy2( - pathlib.Path(__file__).parent / BOARDS_FILE_NAME, project_dir / BOARDS_FILE_NAME - ) + shutil.copy2(BOARDS, project_dir / BOARDS.name) # Place Model Library Format tarball in the special location, which this script uses to decide # whether it's being invoked in a template or generated project. diff --git a/python/tvm/micro/project.py b/python/tvm/micro/project.py index c11ebf707675..5393096b5df3 100644 --- a/python/tvm/micro/project.py +++ b/python/tvm/micro/project.py @@ -103,20 +103,11 @@ def __init__(self, api_client): def _check_project_options(self, options: dict): """Check if options are valid ProjectOptions""" - valid_project_options = [item["name"] for item in self.info()["project_options"]] - valid = True - - if options is not None: - if valid_project_options is None: - valie = False - else: - if not all(element in list(valid_project_options) for element in list(options)): - valid = False - - if not valid: + available_options = [option["name"] for option in self.info()["project_options"]] + if options and not set(options.keys()).issubset(available_options): raise ValueError( - f"""options:{list(options)} include none valid ProjectOptions. - Here is a list of valid options:{list(valid_project_options)}.""" + f"""options:{list(options)} include non valid ProjectOptions. + Here is a list of available options:{list(available_options)}.""" ) def generate_project_from_mlf(self, model_library_format_path, project_dir, options): diff --git a/tests/micro/zephyr/conftest.py b/tests/micro/zephyr/conftest.py index ddd7b0b68f49..177ca8aa269e 100644 --- a/tests/micro/zephyr/conftest.py +++ b/tests/micro/zephyr/conftest.py @@ -17,68 +17,19 @@ import datetime import os import pathlib -import json import pytest -from tvm.micro import project -import tvm.contrib.utils -import tvm.target.target +import test_utils -TEMPLATE_PROJECT_DIR = ( - pathlib.Path(__file__).parent - / ".." - / ".." - / ".." - / "apps" - / "microtvm" - / "zephyr" - / "template_project" -).resolve() - -BOARD_JSON_PATH = TEMPLATE_PROJECT_DIR / "boards.json" - - -def zephyr_boards() -> dict: - """Returns a dict mapping board to target model""" - template = project.TemplateProject.from_directory(TEMPLATE_PROJECT_DIR) - project_options = template.info()["project_options"] - for option in project_options: - if option["name"] == "zephyr_board": - boards = option["choices"] - if option["name"] == "zephyr_model": - models = option["choices"] - - arduino_boards = {boards[i]: models[i] for i in range(len(boards))} - return arduino_boards - - -ZEPHYR_BOARDS = zephyr_boards() - - -def qemu_boards(board: str): - """Returns True if board is QEMU.""" - with open(BOARD_JSON_PATH) as f: - board_properties = json.load(f) - - qemu_boards = [name for name, board in board_properties.items() if board["is_qemu"]] - return board in qemu_boards - - -def has_fpu(board: str): - """Returns True if board has FPU.""" - with open(BOARD_JSON_PATH) as f: - board_properties = json.load(f) - - fpu_boards = [name for name, board in board_properties.items() if board["fpu"]] - return board in fpu_boards +from tvm.contrib.utils import tempdir def pytest_addoption(parser): parser.addoption( "--zephyr-board", required=True, - choices=ZEPHYR_BOARDS.keys(), + choices=test_utils.ZEPHYR_BOARDS.keys(), help=("Zephyr board for test."), ) parser.addoption( @@ -125,4 +76,4 @@ def temp_dir(board): if not os.path.exists(board_workspace.parent): os.makedirs(board_workspace.parent) - return tvm.contrib.utils.tempdir(board_workspace) + return tempdir(board_workspace) diff --git a/tests/micro/zephyr/test_utils.py b/tests/micro/zephyr/test_utils.py new file mode 100644 index 000000000000..54c3de252f8a --- /dev/null +++ b/tests/micro/zephyr/test_utils.py @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import json +import pathlib + + +TEMPLATE_PROJECT_DIR = ( + pathlib.Path(__file__).parent + / ".." + / ".." + / ".." + / "apps" + / "microtvm" + / "zephyr" + / "template_project" +).resolve() + +BOARDS = TEMPLATE_PROJECT_DIR / "boards.json" + + +def zephyr_boards() -> dict: + """Returns a dict mapping board to target model""" + with open(BOARDS) as f: + board_properties = json.load(f) + + boards_model = {board: info["model"] for board, info in board_properties.items()} + return boards_model + + +ZEPHYR_BOARDS = zephyr_boards() + + +def qemu_boards(board: str): + """Returns True if board is QEMU.""" + with open(BOARDS) as f: + board_properties = json.load(f) + + qemu_boards = [name for name, board in board_properties.items() if board["is_qemu"]] + return board in qemu_boards + + +def has_fpu(board: str): + """Returns True if board has FPU.""" + with open(BOARDS) as f: + board_properties = json.load(f) + + fpu_boards = [name for name, board in board_properties.items() if board["fpu"]] + return board in fpu_boards diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py index 6f09b2174d9e..22a101732389 100644 --- a/tests/micro/zephyr/test_zephyr.py +++ b/tests/micro/zephyr/test_zephyr.py @@ -14,14 +14,12 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - import logging import os import pathlib import subprocess import sys import logging -import json import pytest import numpy as np @@ -29,19 +27,12 @@ from PIL import Image import tvm -import tvm.rpc -import tvm.micro -import tvm.testing import tvm.relay as relay from tvm.relay.testing import byoc - from tvm.contrib import utils -from tvm.relay.expr_functor import ExprMutator -from tvm.relay.op.annotation import compiler_begin, compiler_end - from tvm.micro.testing import check_tune_log -import conftest +import test_utils _LOG = logging.getLogger(__name__) @@ -58,21 +49,23 @@ def _make_sess_from_op( def _make_session(temp_dir, zephyr_board, west_cmd, mod, build_config): - stack_size = None - if conftest.qemu_boards(zephyr_board): - stack_size = 1536 + if test_utils.qemu_boards(zephyr_board): + config_main_stack_size = 1536 + + project_options = { + "project_type": "host_driven", + "west_cmd": west_cmd, + "verbose": bool(build_config.get("debug")), + "zephyr_board": zephyr_board, + } + if config_main_stack_size: + project_options["config_main_stack_size"] = config_main_stack_size project = tvm.micro.generate_project( - str(conftest.TEMPLATE_PROJECT_DIR), + str(test_utils.TEMPLATE_PROJECT_DIR), mod, temp_dir / "project", - { - "project_type": "host_driven", - "west_cmd": west_cmd, - "verbose": bool(build_config.get("debug")), - "zephyr_board": zephyr_board, - "main_stack_size": stack_size, - }, + project_options, ) project.build() project.flash() @@ -94,7 +87,7 @@ def _make_add_sess(temp_dir, model, zephyr_board, west_cmd, build_config, dtype= def test_add_uint(temp_dir, board, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" - model = conftest.ZEPHYR_BOARDS[board] + model = test_utils.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -118,8 +111,8 @@ def test_basic_add(sess): @tvm.testing.requires_micro def test_add_float(temp_dir, board, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" - model = conftest.ZEPHYR_BOARDS[board] - if not conftest.has_fpu(board): + model = test_utils.ZEPHYR_BOARDS[board] + if not test_utils.has_fpu(board): pytest.skip(f"FPU not enabled for {board}") build_config = {"debug": tvm_debug} @@ -145,7 +138,7 @@ def test_basic_add(sess): def test_platform_timer(temp_dir, board, west_cmd, tvm_debug): """Test compiling the on-device runtime.""" - model = conftest.ZEPHYR_BOARDS[board] + model = test_utils.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -173,7 +166,7 @@ def test_basic_add(sess): @tvm.testing.requires_micro def test_relay(temp_dir, board, west_cmd, tvm_debug): """Testing a simple relay graph""" - model = conftest.ZEPHYR_BOARDS[board] + model = test_utils.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} shape = (10,) dtype = "int8" @@ -204,7 +197,7 @@ def test_relay(temp_dir, board, west_cmd, tvm_debug): @tvm.testing.requires_micro def test_onnx(temp_dir, board, west_cmd, tvm_debug): """Testing a simple ONNX model.""" - model = conftest.ZEPHYR_BOARDS[board] + model = test_utils.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} this_dir = pathlib.Path(os.path.dirname(__file__)) @@ -281,7 +274,7 @@ def check_result( @tvm.testing.requires_micro def test_byoc_microtvm(temp_dir, board, west_cmd, tvm_debug): """This is a simple test case to check BYOC capabilities of microTVM""" - model = conftest.ZEPHYR_BOARDS[board] + model = test_utils.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} x = relay.var("x", shape=(10, 10)) w0 = relay.var("w0", shape=(10, 10)) @@ -361,7 +354,7 @@ def _make_add_sess_with_shape(temp_dir, model, zephyr_board, west_cmd, shape, bu @tvm.testing.requires_micro def test_rpc_large_array(temp_dir, board, west_cmd, tvm_debug, shape): """Test large RPC array transfer.""" - model = conftest.ZEPHYR_BOARDS[board] + model = test_utils.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} # NOTE: run test in a nested function so cPython will delete arrays before closing the session. @@ -380,9 +373,7 @@ def test_tensors(sess): @tvm.testing.requires_micro def test_autotune_conv2d(temp_dir, board, west_cmd, tvm_debug): """Test AutoTune for microTVM Zephyr""" - import tvm.relay as relay - - model = conftest.ZEPHYR_BOARDS[board] + model = test_utils.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} # Create a Relay model @@ -416,23 +407,21 @@ def test_autotune_conv2d(temp_dir, board, west_cmd, tvm_debug): tasks = tvm.autotvm.task.extract_from_program(mod["main"], {}, target) assert len(tasks) > 0 - repo_root = pathlib.Path( - subprocess.check_output(["git", "rev-parse", "--show-toplevel"], encoding="utf-8").strip() - ) + if test_utils.qemu_boards(board): + config_main_stack_size = 1536 - stack_size = None - if conftest.qemu_boards(board): - stack_size = 1536 + project_options = { + "zephyr_board": board, + "west_cmd": west_cmd, + "verbose": 1, + "project_type": "host_driven", + } + if config_main_stack_size: + project_options["config_main_stack_size"] = config_main_stack_size module_loader = tvm.micro.AutoTvmModuleLoader( - template_project_dir=conftest.TEMPLATE_PROJECT_DIR, - project_options={ - "zephyr_board": board, - "west_cmd": west_cmd, - "verbose": 1, - "project_type": "host_driven", - "main_stack_size": stack_size, - }, + template_project_dir=test_utils.TEMPLATE_PROJECT_DIR, + project_options=project_options, ) timeout = 200 diff --git a/tests/micro/zephyr/test_zephyr_aot.py b/tests/micro/zephyr/test_zephyr_aot.py index b1c26706f006..f03b8ecce6d0 100644 --- a/tests/micro/zephyr/test_zephyr_aot.py +++ b/tests/micro/zephyr/test_zephyr_aot.py @@ -14,7 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - import io import logging import os @@ -28,25 +27,19 @@ import numpy as np import tvm -import tvm.rpc -import tvm.micro from tvm.micro.project_api import server -import tvm.testing import tvm.relay as relay -from tvm.contrib import utils from tvm.contrib.download import download_testdata from tvm.micro.interface_api import generate_c_interface_header -import conftest - -_LOG = logging.getLogger(__name__) +import test_utils def _build_project(temp_dir, zephyr_board, west_cmd, mod, build_config, extra_files_tar=None): project_dir = temp_dir / "project" project = tvm.micro.generate_project( - str(conftest.TEMPLATE_PROJECT_DIR), + str(test_utils.TEMPLATE_PROJECT_DIR), mod, project_dir, { @@ -135,7 +128,7 @@ def test_tflite(temp_dir, board, west_cmd, tvm_debug): ]: pytest.skip(msg="Model does not fit.") - model = conftest.ZEPHYR_BOARDS[board] + model = test_utils.ZEPHYR_BOARDS[board] input_shape = (1, 32, 32, 3) output_shape = (1, 10) build_config = {"debug": tvm_debug} @@ -217,7 +210,7 @@ def test_qemu_make_fail(temp_dir, board, west_cmd, tvm_debug): if board not in ["qemu_x86", "mps2_an521"]: pytest.skip(msg="Only for QEMU targets.") - model = conftest.ZEPHYR_BOARDS[board] + model = test_utils.ZEPHYR_BOARDS[board] build_config = {"debug": tvm_debug} shape = (10,) dtype = "float32"