From 0d42c3012105eb7b9d43a13ee66ad5f1e4360feb Mon Sep 17 00:00:00 2001 From: Ryan OShea <86965113+ArmRyan@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:09:34 +0200 Subject: [PATCH] Add new scripts to build and run unit tests and check version changes in git commits (#137) * Adds requirements.txt file * Creates a new bash script to download dependencies and build/run unit tests * Creates a new bash script to check Date and Revision changes in git commits Change-Id: I163f98516778bf2bec58dbd0614e91d348437acc Signed-off-by: Ryan O'Shea --- .gitignore | 2 + Tests/UnitTest/README.md | 45 ++--- Tests/UnitTest/build_and_run_tests.sh | 250 ++++++++++++++++++++++++++ Tests/UnitTest/requirements.txt | 22 +++ check_version_and_date.sh | 57 ++++++ 5 files changed, 356 insertions(+), 20 deletions(-) create mode 100755 Tests/UnitTest/build_and_run_tests.sh create mode 100644 Tests/UnitTest/requirements.txt create mode 100755 check_version_and_date.sh diff --git a/.gitignore b/.gitignore index dbe787b3..a9583901 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ Tests/UnitTest/TestCases/TestData/*/schema_generated.h Tests/UnitTest/PregeneratedData/* Tests/UnitTest/Output/* Tests/UnitTest/Unity/* +Tests/UnitTest/downloads/* +Tests/UnitTest/build-cortex-m* Documentation/html/* Documentation/DoxyGen/src/history.txt Documentation/DoxyGen/nn.dxy diff --git a/Tests/UnitTest/README.md b/Tests/UnitTest/README.md index 590ce1d3..f9c18a8e 100644 --- a/Tests/UnitTest/README.md +++ b/Tests/UnitTest/README.md @@ -4,10 +4,17 @@ Unit test CMSIS-NN functions on any [Arm Mbed OS](https://os.mbed.com/mbed-os/) The [Unity test framework](http://www.throwtheswitch.org/unity) is used for running the actual unit tests. +For a quick setup, it is reccomended to the helper script targetting the Arm Corstone-300 softwware. See the section " Using FVP based on Arm Corstone-300 software ". + ## Requirements +The following apt packages are required. Replace python venv version with your python version. +``` +sudo apt install ruby-full cmake python3.X-venv pip curl git git-lfs +``` + Python3 is required. -It has been tested with Python 3.6 and it has been tested on Ubuntu 16, 18 and 20. +It has been tested with Python 3.10 and it has been tested on Ubuntu 22.04. Make sure to use the latest pip version before starting. If in a virtual environment just start by upgrading pip. @@ -16,29 +23,15 @@ If in a virtual environment just start by upgrading pip. pip install --upgrade pip ``` -See below for what pip packages are needed. - -### Executing unit tests - -If using the script unittest_targets.py for executing unit tests, the following packages are needed. - -``` -pip install pyserial mbed-ls mbed-cli termcolor mercurial pyelftools==0.29 pyyaml jsonschema jinja2 mbed_host_tests mbed_greentea pycryptodome pyusb cmsis_pack_manager psutil cryptography click cbor -``` - -Python packages mbed-cli and and mbed-ls are command line tools so it should not matter if those are installed under Python2 or Python3. These packages have been tested for Python2, with the following versions: mbed-ls(1.7.9) and mbed-cli(1.10.1). They have also been tested for Python3, with the following versions: mbed-ls(1.7.12) and mbed-cli(1.10.5). Package mercurial is needed for package mbed-cli. - -### Generating new test data - -For generating new test data, the following packages are needed. +After upgrading pip, the requirements file found in Tests/UnitTests can be installed. This contains all +python modules required to run all of the scripts. This will install tensorflow and keras to allow the use of +the generate_test_data.py script. If you have version specific requirements, it is reccomended to install this +requirements.txt in a virtual environment. ``` -pip install numpy packaging tensorflow tf-keras~=2.16 +pip install -r requirements.txt ``` - -For generating new data, the python3 packages tensorflow, numpy and packaging are required. Most unit tests use a Keras generated model for reference. The SVDF unit test use a json template as input for generating a model. To do so flatc compiler is needed and it requires a schema file. - #### Get flatc and schema Note this is only needed for generating SVDF unit tests. @@ -77,6 +70,18 @@ Use the -h flag to get more info. ### Using FVP based on Arm Corstone-300 software +The easiest way to run the unit tests on Corstone-300 is to use the build_and_run_tests.sh script. This script will install required packages, build unit tests and run unit tests. This script has been designed for Linux hosts with both aarch64 and x86_64 architectures. For more help use the '-h' flag on the script. + +Sample usage: +``` +./build_and_run_tests.sh -c cortex-m3,cortex-m7,cortex-m55 -o '-Ofast' +``` +By default the script will download and target gcc. To use arm compiler ensure that arm compilers folder is located in path, export CC and use the -a option on the script. + +Downloaded dependencies including python venv can be found in Tests/UnitTests/downloads. Test elfs can be found in Tests/UnitTests/build-($cpu) directories. + +Otherwise, you can build it manually: + The build for unit tests differs from the build of CMSIS-NN as a [standalone library](https://github.com/ARM-software/CMSIS-NN/blob/main/README.md#building-cmsis-nn-as-a-library) in that, there is a dependency to [CMSIS](https://github.com/ARM-software/CMSIS_5) project for the startup files from CMSIS-Core. This is specified by the mandatory CMSIS_PATH CMake argument. diff --git a/Tests/UnitTest/build_and_run_tests.sh b/Tests/UnitTest/build_and_run_tests.sh new file mode 100755 index 00000000..65121987 --- /dev/null +++ b/Tests/UnitTest/build_and_run_tests.sh @@ -0,0 +1,250 @@ +#!/usr/bin/env bash +# +# SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates +# +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the License); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Version: 1.0 +# Date: 2024-05-21 +# This bash script downloads unit test dependencies, builds unit tests and also runs the tests. + +CPU="cortex-m55" +OPTIMIZATION="-Ofast" +QUIET=0 +BUILD=1 +RUN=1 +SETUP_ENVIRONMENT=1 +USE_ARM_COMPILER=0 +USE_PYTHON_VENV=1 +USE_FVP_FROM_DOWNLOAD=1 +USE_GCC_FROM_DOWNLOAD=1 + +ETHOS_U_CORE_PLATFORM_PATH="" +CMSIS_5_PATH="" + + +usage=" +Helper script to setup, build and run CMSIS-NN unit tests + +args: + -h Display this message. + -c Target cpu. Takes multiple arguments as a comma seperated list. eg cortex-m3,cortex-m7,cortex-m55 (default: cortex-m55) + -o Optimization level. (default: '-Ofast') + -q Quiet mode. This reduces the amount of info printed from building and running cmsis-unit tests. + -b Disable CMake build. Only works with previously built targets. Designed to quickly rerun cpu targets. + -r Disable running the unit tests. Designed to test build only or allow user to manually run individual test cases outside of this script. + -e Disable environment setup. This flag will stop the script from attempting to download dependencies. This is just a quiet mode to reduce print outs. + -a Use Arm Compiler that is previously available on machine. Ensure compiler directory is added to path and export CC. + -p Disable the usage of python venv from download directory. Requires dependencies to be install before calling script. + -f Disable the usage of FVP from download directory. Requires FVP to be in path before calling script. + -u Path to ethos-u-core-platform + -g Disable the usage of GCC that is already from download directory. Requires gcc to be in path before calling script. + -C Path to cmsis 5 + + example usage: $(basename "$0") -c cortex-m3,cortex-m4 -o '-O2' -q +" + +while getopts hc:o:qbreapfu:gC: flag +do + case "${flag}" in + h) echo "${usage}";; + c) CPU=${OPTARG};; + o) OPTIMIZATION=${OPTARG};; + q) QUIET=1;; + b) BUILD=0;; + r) RUN=0;; + e) SETUP_ENVIRONMENT=0;; + a) USE_ARM_COMPILER=1;; + p) USE_PYTHON_VENV=0;; + f) USE_FVP_FROM_DOWNLOAD=0;; + u) ETHOS_U_CORE_PLATFORM_PATH="${OPTARG}";; + g) USE_GCC_FROM_DOWNLOAD=0;; + C) CMSIS_5_PATH="${OPTARG}";; + esac +done + +Setup_Environment() { + echo "++ Downloading Corstone300" + if [[ -d ${WORKING_DIR}/corstone300_download ]]; then + echo "Corstone300 already installed. If you wish to install a new version, please delete the old folder." + else + if [[ ${UNAME_M} == x86_64 ]]; then + CORSTONE_URL=https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-300/FVP_Corstone_SSE-300_11.24_13_Linux64.tgz + elif [[ ${UNAME_M} == aarch64 ]]; then + CORSTONE_URL=https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-300/FVP_Corstone_SSE-300_11.24_13_Linux64_armv8l.tgz + fi + + TEMPFILE=$(mktemp -d)/temp_file + wget ${CORSTONE_URL} -O ${TEMPFILE} >&2 + + TEMPDIR=$(mktemp -d) + tar -C ${TEMPDIR} -xvzf ${TEMPFILE} >&2 + mkdir ${WORKING_DIR}/corstone300_download + ${TEMPDIR}/FVP_Corstone_SSE-300.sh --i-agree-to-the-contained-eula --no-interactive -d ${WORKING_DIR}/corstone300_download >&2 + fi + + echo "++ Downloading GCC" + if [[ -d ${WORKING_DIR}/arm_gcc_download ]]; then + echo "Arm GCC already installed. If you wish to install a new version, please delete the old folder." + else + if [[ ${UNAME_M} == x86_64 ]]; then + GCC_URL="https://developer.arm.com/-/media/Files/downloads/gnu/13.2.rel1/binrel/arm-gnu-toolchain-13.2.rel1-x86_64-arm-none-eabi.tar.xz" + elif [[ ${UNAME_M} == aarch64 ]]; then + GCC_URL="https://developer.arm.com/-/media/Files/downloads/gnu/13.2.rel1/binrel/arm-gnu-toolchain-13.2.rel1-aarch64-arm-none-eabi.tar.xz" + fi + + TEMPFILE=$(mktemp -d)/temp_file + wget ${GCC_URL} -O ${TEMPFILE} >&2 + mkdir ${WORKING_DIR}/arm_gcc_download + + tar -C ${WORKING_DIR}/arm_gcc_download --strip-components=1 -xJf ${TEMPFILE} >&2 + fi + + echo "++ Cloning CMSIS-5" + if [[ -d ${WORKING_DIR}/CMSIS_5 ]]; then + echo "CMSIS-5 already installed. If you wish to install a new version, please delete the old folder." + else + git clone https://github.com/ARM-software/CMSIS_5.git + fi + + echo "++ Cloning Ethos-U core platform" + if [[ -d ${WORKING_DIR}/ethos-u-core-platform ]]; then + echo "Ethos-U core platform already installed. If you wish to install a new version, please delete the old folder." + else + git clone https://review.mlplatform.org/ml/ethos-u/ethos-u-core-platform + fi + + echo "++ Setting up python environment" + if [[ -d ${WORKING_DIR}/cmsis_nn_venv ]]; then + echo "Python venv already installed. If you wish to install a new version, please delete the old folder." + else + python3 -m venv cmsis_nn_venv + source cmsis_nn_venv/bin/activate + pip3 install -r ../requirements.txt + deactivate + fi +} + +Build_Tests() { + echo "++ Building Tests" + if [[ ${QUIET} -eq 0 ]]; then + cmake -S ./ -B build-${cpu}-${compiler} -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DTARGET_CPU=${cpu} -DCMSIS_PATH=${CMSIS_5_PATH} -DCMSIS_OPTIMIZATION_LEVEL=${OPTIMIZATION} + cmake --build build-${cpu}-${compiler}/ + + echo "Built successfully into build-${cpu}-${compiler}" + else + cmake_command=$(cmake -S ./ -B build-${cpu}-${compiler} -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} -DTARGET_CPU=${cpu} -DCMSIS_PATH=${CMSIS_5_PATH} -DCMSIS_OPTIMIZATION_LEVEL=${OPTIMIZATION} 2>&1) + make_command=$(cmake --build build-${cpu}-${compiler}/ 2>&1) + echo "${cmake_command}" > build-${cpu}-${compiler}/cmake_command.txt + echo "${make_command}" > build-${cpu}-${compiler}/make_command.txt + + echo "Built successfully into build-${cpu}-${compiler}" + fi +} + +Run_Tests() { + echo "++ Running Unit Tests" + readarray -d '' tests < <(find ./build-${cpu}-${compiler}/ -iname "*.elf" -print0) + for test in "${tests[@]}" + do + echo "Test: ${test}" + output=$(FVP_Corstone_SSE-300_Ethos-U55 -C mps3_board.uart0.shutdown_on_eot=1 -C mps3_board.visualisation.disable-visualisation=1 -C mps3_board.telnetterminal0.start_telnet=0 -C mps3_board.uart0.out_file="-" -C mps3_board.uart0.unbuffered_output=1 ${test}) + echo "$output" | grep "0 Failures" -vqz + if [[ $? -eq 0 ]]; then + echo "${output}" + echo "${test} failed. Script exiting." + exit 1 + elif [[ ${QUIET} -eq 0 ]]; then + echo "${output}" + fi + done + echo "Tests for ${cpu} ran successfully." +} + +if [[ ${BUILD} -eq 0 && ${RUN} -eq 0 && ${SETUP_ENVIRONMENT} -eq 0 ]]; then + echo "All script functions are disabled. Script will exit and do nothing." + exit 1 +fi + +UNAME_M=$(uname -m) +UNAME_S=$(uname -s) + +if [[ ${UNAME_S} != Linux ]]; then + echo "Error: This script only supports Linux." + exit 1 +fi + +mkdir -p downloads +pushd downloads +WORKING_DIR=$(pwd) + +if [[ ${SETUP_ENVIRONMENT} -eq 1 ]]; then + echo "++ Setting Environment" + Setup_Environment +fi + +if [[ ${USE_PYTHON_VENV} -eq 1 ]]; then + source cmsis_nn_venv/bin/activate +fi + +if [[ -z "${ETHOS_U_CORE_PLATFORM_PATH}" ]]; then + ETHOS_U_CORE_PLATFORM_PATH="${WORKING_DIR}/ethos-u-core-platform" +fi + +if [[ -z "${CMSIS_5_PATH}" ]]; then + CMSIS_5_PATH="${WORKING_DIR}/CMSIS_5" +fi + +popd +IFS=',' read -r -a cpu_array <<< "$CPU" + +if [[ ${BUILD} -eq 1 || ${RUN} -eq 1 ]]; then + for cpu in "${cpu_array[@]}" + do + echo "++ Targetting ${cpu}" + if [[ ${USE_ARM_COMPILER} -eq 1 ]]; then + compiler="arm-compiler" + TOOLCHAIN_FILE=${ETHOS_U_CORE_PLATFORM_PATH}/cmake/toolchain/armclang.cmake + else + if [[ ${USE_GCC_FROM_DOWNLOAD} -eq 1 ]]; then + export PATH=${WORKING_DIR}/arm_gcc_download/bin/:${PATH} + fi + compiler="gcc" + TOOLCHAIN_FILE=${ETHOS_U_CORE_PLATFORM_PATH}/cmake/toolchain/arm-none-eabi-gcc.cmake + fi + + if [[ $USE_FVP_FROM_DOWNLOAD -eq 1 ]]; then + if [[ ${UNAME_M} == x86_64 ]]; then + export PATH=${WORKING_DIR}/corstone300_download/models/Linux64_GCC-9.3/:${PATH} + elif [[ ${UNAME_M} == aarch64 ]]; then + export PATH=${WORKING_DIR}/corstone300_download/models/Linux64_armv8l_GCC-9.3/:${PATH} + fi + fi + + if [[ ${BUILD} -eq 1 ]]; then + Build_Tests + fi + + if [[ ${RUN} -eq 1 ]]; then + Run_Tests + fi + done +fi + +echo "" +echo "++ Tests for ${CPU} ran successfully" +if [[ ${USE_PYTHON_VENV} -eq 1 ]]; then + deactivate +fi \ No newline at end of file diff --git a/Tests/UnitTest/requirements.txt b/Tests/UnitTest/requirements.txt new file mode 100644 index 00000000..27f741f4 --- /dev/null +++ b/Tests/UnitTest/requirements.txt @@ -0,0 +1,22 @@ +pyserial +mbed-ls +mbed-cli +termcolor +mercurial +pyelftools == 0.29 +pyyaml +jsonschema +jinja2 +mbed_host_tests +mbed_greentea +pycryptodome +pyusb +cmsis_pack_manager +psutil +cryptography +click +cbor +numpy +packaging +tensorflow +tf-keras ~= 2.16 \ No newline at end of file diff --git a/check_version_and_date.sh b/check_version_and_date.sh new file mode 100755 index 00000000..4ce482ce --- /dev/null +++ b/check_version_and_date.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# +# SPDX-FileCopyrightText: Copyright 2024 Arm Limited and/or its affiliates +# +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the License); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an AS IS BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Version: 1.0 +# Date: 2024-05-21 +# This bash script checks that files changed in the Source and Include directory have +# modifed the Revision and Date fields below the license in each file. + +SHA=$(git rev-parse origin/upstream/main) +CHANGED_FILES=$(git diff --name-only ${SHA} HEAD -- Source Include) +ANY_FAILURE=0 + +echo "++ Checking if version and date was updated in changed files" +echo " ------ " +for file in ${CHANGED_FILES[@]} +do + diff=$(git diff ${SHA} HEAD ${file}) + # This pattern matches "(+|-) * $DATE: Day Month Year" + echo "$diff" | grep -E '(-|\+)\s*\*\s*\$[Date]+:\s*[0-9]+\s*[A-Za-z]+\s*[0-9]+' -vqz + if [[ $? -eq 0 ]]; then + echo "${file}: FAILED" + ANY_FAILURE=1 + else + # This pattern matches "(+|-) * $REVISION: V.X.X.X" + echo "$diff" | grep -E '(-|\+)\s*\*\s*\$[Revision]+:\s*V\.[0-9]+\.[0-9]+\.[0-9]+' -vqz + if [[ $? -eq 0 ]]; then + echo "${file}: FAILED" + ANY_FAILURE=1 + else + echo "${file}: OK" + fi + fi +done + +if [[ $ANY_FAILURE -eq 1 ]]; then + echo " ------ " + echo "At least one file did not pass the revision and date check. Exiting with failure." + exit 1 +elif [[ $ANY_FAILURE -eq 0 ]]; then + echo " ------ " + echo "All files passed the revision and date check. Exiting successfully." +fi \ No newline at end of file