From 9100a3c975c904ddb68e6519f459d3a2a4e8543c Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Fri, 17 Sep 2021 08:57:17 -0700 Subject: [PATCH] [microTVM][autoTVM] Follow up fixes to #9003 (#9018) * fix test and cleanup * fix tutorial doc * fix verbose for tutorial * fix tune check * address comments * address comments --- python/tvm/micro/session.py | 12 +++++--- python/tvm/micro/testing.py | 33 +++++++++++++++++++++ src/runtime/crt/host/Makefile | 6 +++- src/runtime/crt/host/microtvm_api_server.py | 8 +++-- src/runtime/rpc/rpc_endpoint.cc | 15 +++++----- src/runtime/rpc/rpc_endpoint.h | 8 ++--- tests/micro/zephyr/test_zephyr.py | 13 ++++++-- tests/python/unittest/test_crt.py | 11 ++++--- tutorials/micro/micro_autotune.py | 26 ++++++++++------ 9 files changed, 97 insertions(+), 35 deletions(-) create mode 100644 python/tvm/micro/testing.py diff --git a/python/tvm/micro/session.py b/python/tvm/micro/session.py index ced20b7ebfbf..d545f2e7daa4 100644 --- a/python/tvm/micro/session.py +++ b/python/tvm/micro/session.py @@ -87,6 +87,8 @@ def __init__( self._rpc = None self._graph_executor = None + self._exit_called = False + def get_system_lib(self): return self._rpc.get_function("runtime.SystemLib")() @@ -130,7 +132,7 @@ def __enter__(self): int(timeouts.session_start_retry_timeout_sec * 1e6), int(timeouts.session_start_timeout_sec * 1e6), int(timeouts.session_established_timeout_sec * 1e6), - self._shutdown, + self._cleanup, ) ) self.device = self._rpc.cpu(0) @@ -142,9 +144,11 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, exc_traceback): """Tear down this session and associated RPC session resources.""" - self.transport.__exit__(exc_type, exc_value, exc_traceback) + if not self._exit_called: + self._exit_called = True + self.transport.__exit__(exc_type, exc_value, exc_traceback) - def _shutdown(self): + def _cleanup(self): self.__exit__(None, None, None) @@ -289,6 +293,6 @@ def compile_and_create_micro_session( transport = generated_project.transport() rpc_session = Session(transport_context_manager=transport) - # RPC exit is called by shutdown function. + # RPC exit is called by cleanup function. rpc_session.__enter__() return rpc_session._rpc._sess diff --git a/python/tvm/micro/testing.py b/python/tvm/micro/testing.py new file mode 100644 index 000000000000..124f66e021a3 --- /dev/null +++ b/python/tvm/micro/testing.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Defines the test methods used with microTVM.""" + +import pathlib +import json +from typing import Union + + +def check_tune_log(log_path: Union[pathlib.Path, str]): + """Read the tuning log and check each result.""" + with open(log_path, "r") as f: + lines = f.readlines() + + for line in lines: + if len(line) > 0: + tune_result = json.loads(line) + assert tune_result["result"][0][0] < 1000000000.0 diff --git a/src/runtime/crt/host/Makefile b/src/runtime/crt/host/Makefile index efed3c438699..98a810e0d1b1 100644 --- a/src/runtime/crt/host/Makefile +++ b/src/runtime/crt/host/Makefile @@ -28,7 +28,11 @@ CC ?= ${PREFIX}gcc CXX ?= ${PREFIX}g++ RANLIB ?= ${PREFIX}ranlib +ifeq (${VERBOSE}, 1) +QUIET ?= +else QUIET ?= @ +endif PWD = $(shell pwd) BUILD_DIR = build @@ -38,7 +42,7 @@ CRT_LIBS = $(patsubst %, $(BUILD_DIR)/crt/lib%.a, $(CRT_LIB_NAMES)) CRT_INCLUDES = $(glob crt/include/**) $(BUILD_DIR)/crt/lib%.a: $(glob crt/src/runtime/%/*.c) - ${QUIET}cd crt && $(MAKE) \ + ${QUIET}cd crt && $(MAKE) -s \ BUILD_DIR=../$(BUILD_DIR)/crt \ CRT_CONFIG=$(PWD)/crt_config/crt_config.h \ EXTRA_CFLAGS="$(CFLAGS)" \ diff --git a/src/runtime/crt/host/microtvm_api_server.py b/src/runtime/crt/host/microtvm_api_server.py index 5f9019817e82..546ac1448011 100644 --- a/src/runtime/crt/host/microtvm_api_server.py +++ b/src/runtime/crt/host/microtvm_api_server.py @@ -51,7 +51,11 @@ def server_info_query(self, tvm_version): model_library_format_path="" if IS_TEMPLATE else PROJECT_DIR / MODEL_LIBRARY_FORMAT_RELPATH, - project_options=[server.ProjectOption("verbose", help="Run make with verbose output")], + project_options=[ + server.ProjectOption( + "verbose", help="Run make with verbose output", choices=(True, False) + ) + ], ) # These files and directories will be recursively copied into generated projects from the CRT. @@ -111,7 +115,7 @@ def generate_project(self, model_library_format_path, standalone_crt_dir, projec def build(self, options): args = ["make"] if options.get("verbose"): - args.append("QUIET=") + args.append("VERBOSE=1") args.append(self.BUILD_TARGET) diff --git a/src/runtime/rpc/rpc_endpoint.cc b/src/runtime/rpc/rpc_endpoint.cc index 2f1fc54f39d0..07b90058ce99 100644 --- a/src/runtime/rpc/rpc_endpoint.cc +++ b/src/runtime/rpc/rpc_endpoint.cc @@ -684,20 +684,21 @@ void RPCEndpoint::Init() { /*! * \brief Create a new RPCEndpoint instance. - * \param channel RPCChannel used to communicate + * \param channel RPCChannel used to communicate. * \param name Name of this session, used to identify log messages from this RPCEndpoint instance. - * \param The remote key reported during protocol initialization, or "%toinit" if the RPCEndpoint - * should handle this phase of the protocol for you. Some servers may prefer to access parts of - * the key to modify their behavior. + * \param remote_key The remote key reported during protocol initialization, or "%toinit" if the + * RPCEndpoint should handle this phase of the protocol for you. Some servers may prefer to access + * parts of the key to modify their behavior. + * \param fcleanup The cleanup Packed function. */ std::shared_ptr RPCEndpoint::Create(std::unique_ptr channel, std::string name, std::string remote_key, - TypedPackedFunc fshutdown) { + TypedPackedFunc fcleanup) { std::shared_ptr endpt = std::make_shared(); endpt->channel_ = std::move(channel); endpt->name_ = std::move(name); endpt->remote_key_ = std::move(remote_key); - endpt->fshutdown_ = fshutdown; + endpt->fcleanup_ = fcleanup; endpt->Init(); return endpt; } @@ -736,7 +737,7 @@ void RPCEndpoint::ServerLoop() { (*f)(); } channel_.reset(nullptr); - if (fshutdown_ != nullptr) fshutdown_(); + if (fcleanup_ != nullptr) fcleanup_(); } int RPCEndpoint::ServerAsyncIOEventHandler(const std::string& in_bytes, int event_flag) { diff --git a/src/runtime/rpc/rpc_endpoint.h b/src/runtime/rpc/rpc_endpoint.h index f6784faba0f6..ed19a3f59e58 100644 --- a/src/runtime/rpc/rpc_endpoint.h +++ b/src/runtime/rpc/rpc_endpoint.h @@ -161,13 +161,13 @@ class RPCEndpoint { * \param channel The communication channel. * \param name The local name of the session, used for debug * \param remote_key The remote key of the session - * \param fshutdown The shutdown Packed function * if remote_key equals "%toinit", we need to re-intialize * it by event handler. + * \param fcleanup The cleanup Packed function. */ static std::shared_ptr Create(std::unique_ptr channel, std::string name, std::string remote_key, - TypedPackedFunc fshutdown = nullptr); + TypedPackedFunc fcleanup = nullptr); private: class EventHandler; @@ -192,8 +192,8 @@ class RPCEndpoint { std::string name_; // The remote key std::string remote_key_; - // The shutdown Packed Function - TypedPackedFunc fshutdown_; + // Invoked when the RPC session is terminated + TypedPackedFunc fcleanup_; }; /*! diff --git a/tests/micro/zephyr/test_zephyr.py b/tests/micro/zephyr/test_zephyr.py index d2d5522b1a0a..b6396ce53315 100644 --- a/tests/micro/zephyr/test_zephyr.py +++ b/tests/micro/zephyr/test_zephyr.py @@ -21,6 +21,7 @@ import subprocess import sys import logging +import json import pytest import numpy as np @@ -38,6 +39,8 @@ from tvm.relay.expr_functor import ExprMutator from tvm.relay.op.annotation import compiler_begin, compiler_end +from tvm.micro.testing import check_tune_log + import conftest _LOG = logging.getLogger(__name__) @@ -430,13 +433,18 @@ def test_autotune_conv2d(temp_dir, board, west_cmd, tvm_debug): "project_type": "host_driven", }, ) + + timeout = 200 builder = tvm.autotvm.LocalBuilder( + timeout=timeout, n_parallel=1, build_kwargs={"build_option": {"tir.disable_vectorize": True}}, do_fork=True, build_func=tvm.micro.autotvm_build_func, ) - runner = tvm.autotvm.LocalRunner(number=1, repeat=1, timeout=100, module_loader=module_loader) + runner = tvm.autotvm.LocalRunner( + number=1, repeat=1, timeout=timeout, module_loader=module_loader + ) measure_option = tvm.autotvm.measure_option(builder=builder, runner=runner) @@ -456,8 +464,9 @@ def test_autotune_conv2d(temp_dir, board, west_cmd, tvm_debug): ], si_prefix="M", ) + assert tuner.best_flops > 0 - assert tuner.best_flops > 0 + check_tune_log(log_path) # Build without tuning with pass_context: diff --git a/tests/python/unittest/test_crt.py b/tests/python/unittest/test_crt.py index af14a38c9f9a..62e68ab01ce5 100644 --- a/tests/python/unittest/test_crt.py +++ b/tests/python/unittest/test_crt.py @@ -15,18 +15,14 @@ # specific language governing permissions and limitations # under the License. -import contextlib -import copy -import glob import os import pathlib import pytest import shutil +import json pytest.importorskip("pty") import sys -import subprocess -import textwrap import numpy as np import pytest @@ -39,6 +35,8 @@ from tvm.topi.utils import get_const_tuple from tvm.topi.testing import conv2d_nchw_python +from tvm.micro.testing import check_tune_log + BUILD = True DEBUG = False @@ -285,8 +283,9 @@ def test_autotune(): ], si_prefix="M", ) + assert tuner.best_flops > 0 - assert tuner.best_flops > 0 + check_tune_log(tune_log_file) # Build without tuning with pass_context: diff --git a/tutorials/micro/micro_autotune.py b/tutorials/micro/micro_autotune.py index f89432ff01cf..e7a1fa84a110 100644 --- a/tutorials/micro/micro_autotune.py +++ b/tutorials/micro/micro_autotune.py @@ -20,7 +20,9 @@ Autotuning with micro TVM ========================= -**Author**: `Andrew Reusch `_, `Mehrdad Hessar ` +**Authors**: +`Andrew Reusch `_, +`Mehrdad Hessar `_ This tutorial explains how to autotune a model using the C runtime. """ @@ -117,7 +119,7 @@ module_loader = tvm.micro.AutoTvmModuleLoader( template_project_dir=repo_root / "src" / "runtime" / "crt" / "host", - project_options={}, + project_options={"verbose": False}, ) builder = tvm.autotvm.LocalBuilder( n_parallel=1, @@ -136,7 +138,7 @@ # project_options={ # "zephyr_board": BOARD, # "west_cmd": "west", -# "verbose": 1, +# "verbose": False, # "project_type": "host_driven", # }, # ) @@ -147,8 +149,8 @@ # build_func=tvm.micro.autotvm_build_func, # ) # runner = tvm.autotvm.LocalRunner(number=1, repeat=1, timeout=100, module_loader=module_loader) - -# measure_option = tvm.autotvm.measure_option(builder=builder, runner=runner) +# +# measure_option = tvm.autotvm.measure_option(builder=builder, runner=runner) ################ # Run Autotuning @@ -181,7 +183,10 @@ temp_dir = tvm.contrib.utils.tempdir() project = tvm.micro.generate_project( - str(repo_root / "src" / "runtime" / "crt" / "host"), lowered, temp_dir / "project" + str(repo_root / "src" / "runtime" / "crt" / "host"), + lowered, + temp_dir / "project", + {"verbose": False}, ) # Compiling for physical hardware @@ -193,7 +198,7 @@ # { # "zephyr_board": BOARD, # "west_cmd": "west", -# "verbose": 1, +# "verbose": False, # "project_type": "host_driven", # }, # ) @@ -221,7 +226,10 @@ temp_dir = tvm.contrib.utils.tempdir() project = tvm.micro.generate_project( - str(repo_root / "src" / "runtime" / "crt" / "host"), lowered_tuned, temp_dir / "project" + str(repo_root / "src" / "runtime" / "crt" / "host"), + lowered_tuned, + temp_dir / "project", + {"verbose": False}, ) # Compiling for physical hardware @@ -233,7 +241,7 @@ # { # "zephyr_board": BOARD, # "west_cmd": "west", -# "verbose": 1, +# "verbose": False, # "project_type": "host_driven", # }, # )