Skip to content

Commit

Permalink
[OIS] Add integration tests for Open IoT SDK examples
Browse files Browse the repository at this point in the history
Add network setup script to enable/disable the TAP/TUN network
environment. Create TAP/TUN device and bridge interface to connect it
with the ethernet network interface.

Create Pytest implementation of Open IoT SDK examples integration tests.
Implement lock-app and shell examples tests cases.
Add test command to Open IoT SDK example script.
Add integrations tests to CI workflow.
Add Open IoT SDK examples testing to Vscode tasks.

Signed-off-by: ATmobica <artur.tynecki@arm.com>
  • Loading branch information
ATmobica committed Nov 30, 2022
1 parent 20ce312 commit 8f9e776
Show file tree
Hide file tree
Showing 20 changed files with 1,599 additions and 7 deletions.
23 changes: 23 additions & 0 deletions .github/workflows/examples-openiotsdk.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,17 @@ jobs:
name: Open IoT SDK examples building
timeout-minutes: 90

env:
TEST_NETWORK_NAME: OIStest

runs-on: ubuntu-latest
if: github.actor != 'restyled-io[bot]'

container:
image: connectedhomeip/chip-build-openiotsdk:0.6.06
volumes:
- "/tmp/bloat_reports:/tmp/bloat_reports"
options: --privileged

steps:
- uses: Wandalen/wretry.action@v1.0.36
Expand All @@ -58,6 +62,11 @@ jobs:
timeout-minutes: 10
run: scripts/build/gn_bootstrap.sh

- name: Build and install Python controller
timeout-minutes: 10
run: |
scripts/run_in_build_env.sh './scripts/build_python.sh --install_wheel build-env'
- name: Build shell example
id: build_shell
timeout-minutes: 10
Expand All @@ -77,3 +86,17 @@ jobs:
openiotsdk release lock-app \
examples/lock-app/openiotsdk/build/chip-openiotsdk-lock-app-example.elf \
/tmp/bloat_reports/
- name: Test shell example
if: steps.build_shell.outcome == 'success'
timeout-minutes: 5
run: |
scripts/examples/openiotsdk_example.sh -C test shell
- name: Test lock-app example
if: steps.build_lock_app.outcome == 'success'
timeout-minutes: 5
run: |
scripts/setup/openiotsdk/network_setup.sh -n $TEST_NETWORK_NAME up
scripts/run_in_ns.sh ${TEST_NETWORK_NAME}ns scripts/examples/openiotsdk_example.sh -C test -n ${TEST_NETWORK_NAME}tap lock-app
scripts/setup/openiotsdk/network_setup.sh -n $TEST_NETWORK_NAME down
8 changes: 7 additions & 1 deletion .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -434,7 +434,7 @@
"executable": "./build/chip-openiotsdk-${input:openiotsdkApp}-example.elf",
"armToolchainPath": "${env:ARM_GCC_TOOLCHAIN_PATH}/bin",
"servertype": "external",
"gdbTarget": ":31627", //GDBserver port on FVP
"gdbTarget": "${input:openiotsdkRemoteHost}:31627", //GDBserver port on FVP
"overrideLaunchCommands": ["-enable-pretty-printing"],
"runToEntryPoint": "main",
"preLaunchTask": "Debug Open IoT SDK example",
Expand Down Expand Up @@ -501,6 +501,12 @@
"description": "What Open IoT SDK example do you want to use?",
"options": ["shell", "lock-app"],
"default": "shell"
},
{
"type": "promptString",
"id": "openiotsdkRemoteHost",
"description": "Type the hostname/IP address of external GDB target that you want to connect to. Leave blank for internal GDB server",
"default": ""
}
]
}
43 changes: 41 additions & 2 deletions .vscode/tasks.json
Original file line number Diff line number Diff line change
Expand Up @@ -265,9 +265,33 @@
{
"label": "Run Open IoT SDK example",
"type": "shell",
"command": "scripts/examples/openiotsdk_example.sh",
"command": "scripts/run_in_ns.sh",
"args": [
"${input:openiotsdkNetworkNamespace}",
"scripts/examples/openiotsdk_example.sh",
"-Crun",
"-n${input:openiotsdkNetworkInterface}",
"${input:openiotsdkExample}"
],
"group": "test",
"problemMatcher": {
"pattern": {
"regexp": "^(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)$",
"file": 1,
"line": 2,
"message": 5
}
}
},
{
"label": "Test Open IoT SDK example",
"type": "shell",
"command": "scripts/run_in_ns.sh",
"args": [
"${input:openiotsdkNetworkNamespace}",
"scripts/examples/openiotsdk_example.sh",
"-Ctest",
"-n${input:openiotsdkNetworkInterface}",
"${input:openiotsdkExample}"
],
"group": "test",
Expand All @@ -283,9 +307,12 @@
{
"label": "Debug Open IoT SDK example",
"type": "shell",
"command": "scripts/examples/openiotsdk_example.sh",
"command": "scripts/run_in_ns.sh",
"args": [
"${input:openiotsdkNetworkNamespace}",
"scripts/examples/openiotsdk_example.sh",
"-Crun",
"-n${input:openiotsdkNetworkInterface}",
"-dtrue",
"${input:openiotsdkExample}"
],
Expand Down Expand Up @@ -363,6 +390,18 @@
"options": ["shell", "lock-app"],
"default": "shell"
},
{
"type": "promptString",
"id": "openiotsdkNetworkNamespace",
"description": "Type the network namespace that you want to use. \"default\" means host default network namespace",
"default": "default"
},
{
"type": "promptString",
"id": "openiotsdkNetworkInterface",
"description": "Type the network interface name that you want to use. \"user\" means user network mode",
"default": "user"
},
{
"type": "promptString",
"id": "exampleGlob",
Expand Down
79 changes: 75 additions & 4 deletions scripts/examples/openiotsdk_example.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

# Build and/or run Open IoT SDK examples.

IS_TEST=0
NAME="$(basename "$0")"
HERE="$(dirname "$0")"
CHIP_ROOT="$(realpath "$HERE"/../..)"
Expand All @@ -34,21 +35,25 @@ FVP_BIN=FVP_Corstone_SSE-300_Ethos-U55
GDB_PLUGIN="$FAST_MODEL_PLUGINS_PATH/GDBRemoteConnection.so"
OIS_CONFIG="$CHIP_ROOT/config/openiotsdk"
FVP_CONFIG_FILE="$OIS_CONFIG/fvp/cs300.conf"
EXAMPLE_TEST_PATH="$CHIP_ROOT/src/test_driver/openiotsdk/integration-tests"
TELNET_TERMINAL_PORT=5000
FAILED_TESTS=0
FVP_NETWORK="user"

function show_usage() {
cat <<EOF
Usage: $0 [options] example
Build and/or run the Open IoT SDK example.
Build, run or test the Open IoT SDK example.
Options:
-h,--help Show this help
-c,--clean Clean target build
-s,--scratch Remove build directory at all before building
-C,--command <command> Action to execute <build-run | run | build - default>
-C,--command <command> Action to execute <build-run | run | test | build - default>
-d,--debug <debug_enable> Build in debug mode <true | false - default>
-p,--path <build_path> Build path <build_path - default is example_dir/build>
-n,--network <network_name> FVP network interface name <network_name - default is "user" which means user network mode>
Examples:
shell
Expand Down Expand Up @@ -122,6 +127,12 @@ function run_fvp() {
RUN_OPTIONS+=(--allow-debug-plugin --plugin "$GDB_PLUGIN")
fi

if [[ $FVP_NETWORK == "user" ]]; then
RUN_OPTIONS+=(-C mps3_board.hostbridge.userNetworking=1)
else
RUN_OPTIONS+=(-C mps3_board.hostbridge.interfaceName="$FVP_NETWORK")
fi

echo "Running $EXAMPLE_EXE_PATH with options: ${RUN_OPTIONS[@]}"

"$FVP_BIN" "${RUN_OPTIONS[@]}" -f "$FVP_CONFIG_FILE" --application "$EXAMPLE_EXE_PATH" >/dev/null 2>&1 &
Expand All @@ -134,6 +145,53 @@ function run_fvp() {
sleep 1
}

function run_test() {

EXAMPLE_EXE_PATH="$BUILD_PATH/chip-openiotsdk-$EXAMPLE-example.elf"
# Check if executable file exists
if ! [ -f "$EXAMPLE_EXE_PATH" ]; then
echo "Error: $EXAMPLE_EXE_PATH does not exist." >&2
exit 1
fi

# Check if FVP exists
if ! [ -x "$(command -v "$FVP_BIN")" ]; then
echo "Error: $FVP_BIN not installed." >&2
exit 1
fi

# Activate Matter environment with pytest
source "$CHIP_ROOT"/scripts/activate.sh

# Check if pytest exists
if ! [ -x "$(command -v pytest)" ]; then
echo "Error: pytest not installed." >&2
exit 1
fi

TEST_OPTIONS=()

if [[ $FVP_NETWORK ]]; then
TEST_OPTIONS+=(--networkInterface="$FVP_NETWORK")
fi

if [[ -f $EXAMPLE_TEST_PATH/$EXAMPLE/test_report.json ]]; then
rm -rf "$EXAMPLE_TEST_PATH/$EXAMPLE"/test_report.json
fi

set +e
pytest --json-report --json-report-summary --json-report-file="$EXAMPLE_TEST_PATH/$EXAMPLE"/test_report.json --binaryPath="$EXAMPLE_EXE_PATH" --fvp="$FVP_BIN" --fvpConfig="$FVP_CONFIG_FILE" "${TEST_OPTIONS[@]}" "$EXAMPLE_TEST_PATH/$EXAMPLE"/test_app.py
set -e

if [[ ! -f $EXAMPLE_TEST_PATH/$EXAMPLE/test_report.json ]]; then
exit 1
else
if [[ $(jq '.summary | has("failed")' $EXAMPLE_TEST_PATH/$EXAMPLE/test_report.json) == true ]]; then
FAILED_TESTS=$(jq '.summary.failed' "$EXAMPLE_TEST_PATH/$EXAMPLE"/test_report.json)
fi
fi
}

SHORT=C:,p:,d:.n:,c,s,h
LONG=command:,path:,debug:.network:,clean,scratch,help
OPTS=$(getopt -n build --options "$SHORT" --longoptions "$LONG" -- "$@")
Expand Down Expand Up @@ -166,6 +224,10 @@ while :; do
BUILD_PATH=$CHIP_ROOT/$2
shift 2
;;
-n | --network)
FVP_NETWORK=$2
shift 2
;;
-* | --*)
shift
break
Expand All @@ -174,7 +236,7 @@ while :; do
echo "Unexpected option: $1"
show_usage
exit 2
;;
;;
esac
done

Expand All @@ -195,7 +257,7 @@ case "$1" in
esac

case "$COMMAND" in
build | run | build-run) ;;
build | run | test | build-run) ;;
*)
echo "Wrong command definition"
show_usage
Expand All @@ -217,3 +279,12 @@ fi
if [[ "$COMMAND" == *"run"* ]]; then
run_fvp
fi

if [[ "$COMMAND" == *"test"* ]]; then
IS_TEST=1
run_test
fi

if [[ $IS_TEST -eq 1 ]]; then
exit "$FAILED_TESTS"
fi
61 changes: 61 additions & 0 deletions scripts/run_in_ns.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
#!/bin/bash

#
# Copyright (c) 2022 Project CHIP Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# This script executes the command given as an argument within
# CHIP_ROOT in a specific network namespace.

NETWORK_NAMESPACE="default"

function show_usage() {
cat <<EOF
Usage: $0 namespace command
Execute the command given as an argument in a specific network namespace.
Example:
scripts/run_in_ns.sh MatterNet "ip a"
Use "default" as a namespace argument to use the host network namespace directly.
EOF
}

if [[ $# -lt 2 ]]; then
show_usage >&2
exit 1
fi

NETWORK_NAMESPACE=$1
shift

if [[ $NETWORK_NAMESPACE == "default" ]]; then
"$@"
else
if [ ! -f /var/run/netns/"$NETWORK_NAMESPACE" ]; then
echo "$NETWORK_NAMESPACE network namespace does not exist"
show_usage >&2
exit 1
fi
echo "Run command: $@ in $NETWORK_NAMESPACE namespace"
if [ "$EUID" -ne 0 ]; then
sudo env PATH="$PATH" ip netns exec "$NETWORK_NAMESPACE" "$@"
else
ip netns exec "$NETWORK_NAMESPACE" "$@"
fi
fi
Loading

0 comments on commit 8f9e776

Please sign in to comment.