From a59f25cac5d2525b583efbb029e6451cdfcc6a69 Mon Sep 17 00:00:00 2001 From: Gavin Uberti Date: Fri, 27 Aug 2021 10:45:43 -0700 Subject: [PATCH 1/4] Fix Arduino DLDevice includes --- apps/microtvm/arduino/example_project/src/model.c | 5 ++--- apps/microtvm/arduino/host_driven/src/model_support.c | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/microtvm/arduino/example_project/src/model.c b/apps/microtvm/arduino/example_project/src/model.c index 9e7c47f75160..b32468687099 100644 --- a/apps/microtvm/arduino/example_project/src/model.c +++ b/apps/microtvm/arduino/example_project/src/model.c @@ -18,13 +18,12 @@ */ #include "model.h" - -#include "Arduino.h" +#include "standalone_crt/include/dlpack/dlpack.h" #include "standalone_crt/include/tvm/runtime/crt/stack_allocator.h" +#include "Arduino.h" // AOT memory array static uint8_t g_aot_memory[WORKSPACE_SIZE]; -extern tvm_model_t tvmgen_default_network; tvm_workspace_t app_workspace; // Blink code for debugging purposes diff --git a/apps/microtvm/arduino/host_driven/src/model_support.c b/apps/microtvm/arduino/host_driven/src/model_support.c index dfcb031136c5..bcc9a109cace 100644 --- a/apps/microtvm/arduino/host_driven/src/model_support.c +++ b/apps/microtvm/arduino/host_driven/src/model_support.c @@ -17,6 +17,8 @@ * under the License. */ +#include "standalone_crt/include/dlpack/dlpack.h" +#include "standalone_crt/include/tvm/runtime/crt/error_codes.h" #include "stdarg.h" // Blink code for debugging purposes From 69b01ac7c3a82af96908f4b858466382510d7b8e Mon Sep 17 00:00:00 2001 From: Gavin Uberti Date: Fri, 27 Aug 2021 10:54:47 -0700 Subject: [PATCH 2/4] microtvm_api_server fails if commands fail --- apps/microtvm/arduino/example_project/src/model.c | 3 ++- .../arduino/template_project/microtvm_api_server.py | 8 +++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/apps/microtvm/arduino/example_project/src/model.c b/apps/microtvm/arduino/example_project/src/model.c index b32468687099..553665191b14 100644 --- a/apps/microtvm/arduino/example_project/src/model.c +++ b/apps/microtvm/arduino/example_project/src/model.c @@ -18,9 +18,10 @@ */ #include "model.h" + +#include "Arduino.h" #include "standalone_crt/include/dlpack/dlpack.h" #include "standalone_crt/include/tvm/runtime/crt/stack_allocator.h" -#include "Arduino.h" // AOT memory array static uint8_t g_aot_memory[WORKSPACE_SIZE]; diff --git a/apps/microtvm/arduino/template_project/microtvm_api_server.py b/apps/microtvm/arduino/template_project/microtvm_api_server.py index 91beaf558249..f41f2c298e34 100644 --- a/apps/microtvm/arduino/template_project/microtvm_api_server.py +++ b/apps/microtvm/arduino/template_project/microtvm_api_server.py @@ -365,7 +365,7 @@ def build(self, options): compile_cmd.append("--verbose") # Specify project to compile - subprocess.run(compile_cmd) + subprocess.run(compile_cmd, check=True) BOARD_LIST_HEADERS = ("Port", "Type", "Board Name", "FQBN", "Core") @@ -402,7 +402,9 @@ def _parse_boards_tabular_str(self, tabular_str): def _auto_detect_port(self, options): list_cmd = [options["arduino_cli_cmd"], "board", "list"] - list_cmd_output = subprocess.run(list_cmd, stdout=subprocess.PIPE).stdout.decode("utf-8") + list_cmd_output = subprocess.run( + list_cmd, check=True, stdout=subprocess.PIPE + ).stdout.decode("utf-8") desired_fqbn = self._get_fqbn(options) for line in self._parse_boards_tabular_str(list_cmd_output): @@ -439,7 +441,7 @@ def flash(self, options): if options.get("verbose"): upload_cmd.append("--verbose") - subprocess.run(upload_cmd) + subprocess.run(upload_cmd, check=True) def open_transport(self, options): # Zephyr example doesn't throw an error in this case From 3a36816a7bf569aa633afa73f3d7dc73f84f07a0 Mon Sep 17 00:00:00 2001 From: Gavin Uberti Date: Fri, 27 Aug 2021 12:06:42 -0700 Subject: [PATCH 3/4] Add regression test for microtvm_api_server not failing --- tests/micro/arduino/test_arduino_workflow.py | 44 +++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/tests/micro/arduino/test_arduino_workflow.py b/tests/micro/arduino/test_arduino_workflow.py index 101d36f9bd2d..4f16c486142c 100644 --- a/tests/micro/arduino/test_arduino_workflow.py +++ b/tests/micro/arduino/test_arduino_workflow.py @@ -19,12 +19,14 @@ import pathlib import shutil import sys +import tempfile import pytest -import tvm -from tvm import micro, relay import conftest +import tvm +from tvm import micro, relay +from tvm.micro.project_api.server import ServerError """ This unit test simulates a simple user workflow, where we: @@ -49,6 +51,30 @@ def project_dir(workspace_dir): return workspace_dir / "project" +# Saves the Arduino project's state, runs the test, then resets it +@pytest.fixture(scope="function") +def does_not_affect_state(project_dir): + with tempfile.TemporaryDirectory() as temp_dir: + prev_project_state = pathlib.Path(temp_dir) + shutil.copytree(project_dir, prev_project_state / "project") + yield + + # We can't delete project_dir or it'll mess up the Arduino CLI working directory + # Instead, delete everything in project_dir, and then copy over the files + for item in project_dir.iterdir(): + if item.is_dir(): + shutil.rmtree(item) + else: + item.unlink() # Delete file + # Once we upgrade to Python 3.7, this can be replaced with + # shutil.copytree(dirs_exist_ok=True) + for item in (prev_project_state / "project").iterdir(): + if item.is_dir(): + shutil.copytree(item, project_dir / item.name) + else: + shutil.copy2(item, project_dir) + + def _generate_project(arduino_board, arduino_cli_cmd, workspace_dir, mod, build_config): return tvm.micro.generate_project( str(conftest.TEMPLATE_PROJECT_DIR), @@ -135,6 +161,20 @@ def test_import_rerouting(project_dir, project): assert "include/tvm/runtime/crt/platform.h" in c_backend_api_c +@pytest.mark.usefixtures("does_not_affect_state") +def test_blank_project_compiles(project): + project.build() + + +# Add a bug (an extra curly brace) and make sure the project doesn't compile +@pytest.mark.usefixtures("does_not_affect_state") +def test_bugged_project_compile_fails(project_dir, project): + with open(project_dir / "project.ino", "a") as main_file: + main_file.write("}\n") + with pytest.raises(ServerError): + project.build() + + # Build on top of the generated project by replacing the # top-level .ino fileand adding data input files, much # like a user would From 2e433f684533f7bce63b14be823a91da3072881d Mon Sep 17 00:00:00 2001 From: Gavin Uberti Date: Mon, 30 Aug 2021 12:02:25 -0700 Subject: [PATCH 4/4] Address PR comments Break error detection tests into separate file Address comments from Mousius Re-add necessary fixture --- tests/micro/arduino/conftest.py | 40 +++++++++ .../arduino/test_arduino_error_detection.py | 52 +++++++++++ .../micro/arduino/test_arduino_rpc_server.py | 4 +- tests/micro/arduino/test_arduino_workflow.py | 86 +------------------ 4 files changed, 97 insertions(+), 85 deletions(-) create mode 100644 tests/micro/arduino/test_arduino_error_detection.py diff --git a/tests/micro/arduino/conftest.py b/tests/micro/arduino/conftest.py index bcb2bddf2cab..f06171d1b8e1 100644 --- a/tests/micro/arduino/conftest.py +++ b/tests/micro/arduino/conftest.py @@ -20,6 +20,7 @@ import pytest import tvm.target.target +from tvm import micro, relay # The models that should pass this configuration. Maps a short, identifying platform string to # (model, zephyr_board). @@ -121,3 +122,42 @@ def make_workspace_dir(test_name, platform): t = tvm.contrib.utils.tempdir(board_workspace) # time.sleep(200) return t + + +def make_kws_project(platform, arduino_cli_cmd, tvm_debug, workspace_dir): + this_dir = pathlib.Path(__file__).parent + model, arduino_board = PLATFORMS[platform] + build_config = {"debug": tvm_debug} + + with open(this_dir.parent / "testdata" / "kws" / "yes_no.tflite", "rb") as f: + tflite_model_buf = f.read() + + # TFLite.Model.Model has changed to TFLite.Model from 1.14 to 2.1 + try: + import tflite.Model + + tflite_model = tflite.Model.Model.GetRootAsModel(tflite_model_buf, 0) + except AttributeError: + import tflite + + tflite_model = tflite.Model.GetRootAsModel(tflite_model_buf, 0) + + mod, params = relay.frontend.from_tflite(tflite_model) + target = tvm.target.target.micro( + model, options=["--link-params=1", "--unpacked-api=1", "--executor=aot"] + ) + + with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): + mod = relay.build(mod, target, params=params) + + return tvm.micro.generate_project( + str(TEMPLATE_PROJECT_DIR), + mod, + workspace_dir / "project", + { + "arduino_board": arduino_board, + "arduino_cli_cmd": arduino_cli_cmd, + "project_type": "example_project", + "verbose": bool(build_config.get("debug")), + }, + ) diff --git a/tests/micro/arduino/test_arduino_error_detection.py b/tests/micro/arduino/test_arduino_error_detection.py new file mode 100644 index 000000000000..1789fff2e7bb --- /dev/null +++ b/tests/micro/arduino/test_arduino_error_detection.py @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pathlib +import re +import sys + +import pytest + +import conftest +from tvm.micro.project_api.server import ServerError + + +# A new project and workspace dir is created for EVERY test +@pytest.fixture +def workspace_dir(request, platform): + return conftest.make_workspace_dir("arduino_error_detection", platform) + + +@pytest.fixture +def project(platform, arduino_cli_cmd, tvm_debug, workspace_dir): + return conftest.make_kws_project(platform, arduino_cli_cmd, tvm_debug, workspace_dir) + + +def test_blank_project_compiles(workspace_dir, project): + project.build() + + +# Add a bug (an extra curly brace) and make sure the project doesn't compile +def test_bugged_project_compile_fails(workspace_dir, project): + with open(workspace_dir / "project" / "project.ino", "a") as main_file: + main_file.write("}\n") + with pytest.raises(ServerError): + project.build() + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__] + sys.argv[1:])) diff --git a/tests/micro/arduino/test_arduino_rpc_server.py b/tests/micro/arduino/test_arduino_rpc_server.py index 1b165a02e9d1..57ebbc605197 100644 --- a/tests/micro/arduino/test_arduino_rpc_server.py +++ b/tests/micro/arduino/test_arduino_rpc_server.py @@ -37,8 +37,8 @@ import conftest -# We'll make a new workspace for each test -@pytest.fixture(scope="function") +# # A new project and workspace dir is created for EVERY test +@pytest.fixture def workspace_dir(platform): return conftest.make_workspace_dir("arduino_rpc_server", platform) diff --git a/tests/micro/arduino/test_arduino_workflow.py b/tests/micro/arduino/test_arduino_workflow.py index 4f16c486142c..8980ef321c8c 100644 --- a/tests/micro/arduino/test_arduino_workflow.py +++ b/tests/micro/arduino/test_arduino_workflow.py @@ -19,14 +19,10 @@ import pathlib import shutil import sys -import tempfile import pytest import conftest -import tvm -from tvm import micro, relay -from tvm.micro.project_api.server import ServerError """ This unit test simulates a simple user workflow, where we: @@ -40,7 +36,8 @@ """ -# Since these tests are sequential, we'll use the same project for all tests +# Since these tests are sequential, we'll use the same project/workspace +# directory for all tests in this file @pytest.fixture(scope="module") def workspace_dir(request, platform): return conftest.make_workspace_dir("arduino_workflow", platform) @@ -51,73 +48,10 @@ def project_dir(workspace_dir): return workspace_dir / "project" -# Saves the Arduino project's state, runs the test, then resets it -@pytest.fixture(scope="function") -def does_not_affect_state(project_dir): - with tempfile.TemporaryDirectory() as temp_dir: - prev_project_state = pathlib.Path(temp_dir) - shutil.copytree(project_dir, prev_project_state / "project") - yield - - # We can't delete project_dir or it'll mess up the Arduino CLI working directory - # Instead, delete everything in project_dir, and then copy over the files - for item in project_dir.iterdir(): - if item.is_dir(): - shutil.rmtree(item) - else: - item.unlink() # Delete file - # Once we upgrade to Python 3.7, this can be replaced with - # shutil.copytree(dirs_exist_ok=True) - for item in (prev_project_state / "project").iterdir(): - if item.is_dir(): - shutil.copytree(item, project_dir / item.name) - else: - shutil.copy2(item, project_dir) - - -def _generate_project(arduino_board, arduino_cli_cmd, workspace_dir, mod, build_config): - return tvm.micro.generate_project( - str(conftest.TEMPLATE_PROJECT_DIR), - mod, - workspace_dir / "project", - { - "arduino_board": arduino_board, - "arduino_cli_cmd": arduino_cli_cmd, - "project_type": "example_project", - "verbose": bool(build_config.get("debug")), - }, - ) - - # We MUST pass workspace_dir, not project_dir, or the workspace will be dereferenced too soon @pytest.fixture(scope="module") def project(platform, arduino_cli_cmd, tvm_debug, workspace_dir): - this_dir = pathlib.Path(__file__).parent - model, arduino_board = conftest.PLATFORMS[platform] - build_config = {"debug": tvm_debug} - - with open(this_dir.parent / "testdata" / "kws" / "yes_no.tflite", "rb") as f: - tflite_model_buf = f.read() - - # TFLite.Model.Model has changed to TFLite.Model from 1.14 to 2.1 - try: - import tflite.Model - - tflite_model = tflite.Model.Model.GetRootAsModel(tflite_model_buf, 0) - except AttributeError: - import tflite - - tflite_model = tflite.Model.GetRootAsModel(tflite_model_buf, 0) - - mod, params = relay.frontend.from_tflite(tflite_model) - target = tvm.target.target.micro( - model, options=["--link-params=1", "--unpacked-api=1", "--executor=aot"] - ) - - with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}): - mod = relay.build(mod, target, params=params) - - return _generate_project(arduino_board, arduino_cli_cmd, workspace_dir, mod, build_config) + return conftest.make_kws_project(platform, arduino_cli_cmd, tvm_debug, workspace_dir) def _get_directory_elements(directory): @@ -161,20 +95,6 @@ def test_import_rerouting(project_dir, project): assert "include/tvm/runtime/crt/platform.h" in c_backend_api_c -@pytest.mark.usefixtures("does_not_affect_state") -def test_blank_project_compiles(project): - project.build() - - -# Add a bug (an extra curly brace) and make sure the project doesn't compile -@pytest.mark.usefixtures("does_not_affect_state") -def test_bugged_project_compile_fails(project_dir, project): - with open(project_dir / "project.ino", "a") as main_file: - main_file.write("}\n") - with pytest.raises(ServerError): - project.build() - - # Build on top of the generated project by replacing the # top-level .ino fileand adding data input files, much # like a user would