[OIS] Add Open IoT SDK unit-tests

Adaptation changes:
Exclude LwIP initialization for Open IoT SDK platform in unit tests
implementation.
Skip read/write chunking tests in controller component.
Skip CommissionerDUTVectors test in credentials component.
Skip ControllerTests - project-chip#23747 issue
Skip CHIP stack initialization in FailSafeContext test.

Add Open IoT SDK platform unit-tests application in
src/test_driver/openiotsdk/unit-tests directory.
Add unit tests to Open IoT SDK example script, CI workflow
and VScode tasks.
Add unit-tests application documentation.

Signed-off-by: ATmobica <artur.tynecki@arm.com>

.github/workflows/examples-openiotsdk.yaml

@@ -26,7 +26,7 @@ concurrency:
 jobs:
     openiotsdk:
         name: Open IoT SDK examples building
-        timeout-minutes: 90
+        timeout-minutes: 140
 
         env:
             TEST_NETWORK_NAME: OIStest
@@ -87,6 +87,12 @@ jobs:
                     examples/lock-app/openiotsdk/build/chip-openiotsdk-lock-app-example.elf \
                     /tmp/bloat_reports/
 
+            - name: Build unit tests
+              id: build_unit_tests
+              timeout-minutes: 10
+              run: |
+                  scripts/examples/openiotsdk_example.sh unit-tests
+
             - name: Test shell example
               if: steps.build_shell.outcome == 'success'
               timeout-minutes: 5
@@ -100,3 +106,9 @@ jobs:
                   scripts/setup/openiotsdk/network_setup.sh -n $TEST_NETWORK_NAME up
                   scripts/run_in_ns.sh ${TEST_NETWORK_NAME}ns scripts/examples/openiotsdk_example.sh -C test -n ${TEST_NETWORK_NAME}tap lock-app
                   scripts/setup/openiotsdk/network_setup.sh -n $TEST_NETWORK_NAME down
+            
+            - name: Run unit tests
+              if: steps.build_unit_tests.outcome == 'success' && github.event_name == 'pull_request'
+              timeout-minutes: 90
+              run: |
+                  scripts/examples/openiotsdk_example.sh -C run unit-tests

.vscode/launch.json

@@ -439,6 +439,20 @@
             "runToEntryPoint": "main",
             "preLaunchTask": "Debug Open IoT SDK example",
             "showDevDebugOutput": "parsed"
+        },
+        {
+            "name": "Debug Open IoT SDK unit-tests application",
+            "type": "cortex-debug",
+            "request": "launch",
+            "cwd": "${workspaceRoot}/src/test_driver/openiotsdk/unit-tests",
+            "executable": "./build/${input:openiotsdkUnittest}.elf",
+            "armToolchainPath": "${env:ARM_GCC_TOOLCHAIN_PATH}/bin",
+            "servertype": "external",
+            "gdbTarget": ":31627", //GDBserver port on FVP
+            "overrideLaunchCommands": ["-enable-pretty-printing"],
+            "runToEntryPoint": "main",
+            "preLaunchTask": "Debug Open IoT SDK unit-tests",
+            "showDevDebugOutput": "parsed"
         }
     ],
     "inputs": [
@@ -507,6 +521,39 @@
             "id": "openiotsdkRemoteHost",
             "description": "Type the hostname/IP address of external GDB target that you want to connect to. Leave blank for internal GDB server",
             "default": ""
+        },
+        {
+            "type": "pickString",
+            "id": "openiotsdkUnittest",
+            "description": "What Open IoT SDK unit test do you want to use?",
+            "options": [
+                "accesstest",
+                "AppTests",
+                "ASN1Tests",
+                "BDXTests",
+                "ChipCryptoTests",
+                "ControllerTests",
+                "CoreTests",
+                "CredentialsTest",
+                "DataModelTests",
+                "InetLayerTests",
+                "MdnsTests",
+                "MessagingLayerTests",
+                "MinimalMdnsCoreTests",
+                "MinimalMdnsRecordsTests",
+                "MinimalMdnsRespondersTests",
+                "PlatformTests",
+                "RawTransportTests",
+                "RetransmitTests",
+                "SecureChannelTests",
+                "SetupPayloadTests",
+                "SupportTests",
+                "SystemLayerTests",
+                "TestShell",
+                "TransportLayerTests",
+                "UserDirectedCommissioningTests"
+            ],
+            "default": "accesstest"
         }
     ]
 }

.vscode/tasks.json

@@ -262,6 +262,21 @@
                 }
             }
         },
+        {
+            "label": "Build Open IoT SDK unit-tests",
+            "type": "shell",
+            "command": "scripts/examples/openiotsdk_example.sh",
+            "args": ["-Cbuild", "-d${input:openiotsdkDebugMode}", "unit-tests"],
+            "group": "build",
+            "problemMatcher": {
+                "pattern": {
+                    "regexp": "^(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)$",
+                    "file": 1,
+                    "line": 2,
+                    "message": 5
+                }
+            }
+        },
         {
             "label": "Run Open IoT SDK example",
             "type": "shell",
@@ -283,6 +298,21 @@
                 }
             }
         },
+        {
+            "label": "Run Open IoT SDK unit-tests",
+            "type": "shell",
+            "command": "scripts/examples/openiotsdk_example.sh",
+            "args": ["-Crun", "unit-tests", "${input:openiotsdkUnitTest}"],
+            "group": "test",
+            "problemMatcher": {
+                "pattern": {
+                    "regexp": "^(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)$",
+                    "file": 1,
+                    "line": 2,
+                    "message": 5
+                }
+            }
+        },
         {
             "label": "Test Open IoT SDK example",
             "type": "shell",
@@ -331,6 +361,32 @@
                     "endsPattern": "^.*Connected to localhost*"
                 }
             }
+        },
+        {
+            "label": "Debug Open IoT SDK unit-tests",
+            "type": "shell",
+            "command": "scripts/examples/openiotsdk_example.sh",
+            "args": [
+                "-Crun",
+                "-dtrue",
+                "unit-tests",
+                "${input:openiotsdkUnitTest}"
+            ],
+            "group": "none",
+            "isBackground": true,
+            "problemMatcher": {
+                "pattern": {
+                    "regexp": "^(.*):(\\d+):(\\d+):\\s+(warning|error):\\s+(.*)$",
+                    "file": 1,
+                    "line": 2,
+                    "message": 5
+                },
+                "background": {
+                    "activeOnStart": true,
+                    "beginsPattern": "^.*Trying*",
+                    "endsPattern": "^.*Connected to localhost*"
+                }
+            }
         }
     ],
     "inputs": [
@@ -390,6 +446,40 @@
             "options": ["shell", "lock-app"],
             "default": "shell"
         },
+        {
+            "type": "pickString",
+            "id": "openiotsdkUnitTest",
+            "description": "What unit test do you want to use?",
+            "options": [
+                "all",
+                "accesstest",
+                "AppTests",
+                "ASN1Tests",
+                "BDXTests",
+                "ChipCryptoTests",
+                "ControllerTests",
+                "CoreTests",
+                "CredentialsTest",
+                "DataModelTests",
+                "InetLayerTests",
+                "MdnsTests",
+                "MessagingLayerTests",
+                "MinimalMdnsCoreTests",
+                "MinimalMdnsRecordsTests",
+                "MinimalMdnsRespondersTests",
+                "PlatformTests",
+                "RawTransportTests",
+                "RetransmitTests",
+                "SecureChannelTests",
+                "SetupPayloadTests",
+                "SupportTests",
+                "SystemLayerTests",
+                "TestShell",
+                "TransportLayerTests",
+                "UserDirectedCommissioningTests"
+            ],
+            "default": "all"
+        },
         {
             "type": "promptString",
             "id": "openiotsdkNetworkNamespace",

scripts/examples/openiotsdk_example.sh

@@ -40,11 +40,13 @@ TELNET_TERMINAL_PORT=5000
 FAILED_TESTS=0
 FVP_NETWORK="user"
 
+readarray -t TEST_NAMES <"$CHIP_ROOT"/src/test_driver/openiotsdk/unit-tests/testnames.txt
+
 function show_usage() {
     cat <<EOF
-Usage: $0 [options] example
+Usage: $0 [options] example [test_name]
 
-Build, run or test the Open IoT SDK example.
+Build, run or test the Open IoT SDK examples and unit-tests.
 
 Options:
     -h,--help                       Show this help
@@ -58,6 +60,18 @@ Options:
 Examples:
     shell
     lock-app
+    unit-tests
+
+You can run individual test suites of unit tests by using their names [test_name] with the run command:
+
+EOF
+    cat "$CHIP_ROOT"/src/test_driver/openiotsdk/unit-tests/testnames.txt
+    echo ""
+    cat <<EOF
+Or you can use all tests suites with <all> parameter as [test_name]
+
+The "test" command can be used for all supported examples expect the unit-tests.
+
 EOF
 }
 
@@ -112,7 +126,11 @@ function run_fvp() {
         exit 1
     fi
 
-    EXAMPLE_EXE_PATH="$BUILD_PATH/chip-openiotsdk-$EXAMPLE-example.elf"
+    if [[ $IS_TEST -eq 0 ]]; then
+        EXAMPLE_EXE_PATH="$BUILD_PATH/chip-openiotsdk-$EXAMPLE-example.elf"
+    else
+        EXAMPLE_EXE_PATH="$BUILD_PATH/$EXAMPLE.elf"
+    fi
 
     # Check if executable file exists
     if ! [ -f "$EXAMPLE_EXE_PATH" ]; then
@@ -138,10 +156,30 @@ function run_fvp() {
     "$FVP_BIN" "${RUN_OPTIONS[@]}" -f "$FVP_CONFIG_FILE" --application "$EXAMPLE_EXE_PATH" >/dev/null 2>&1 &
     FVP_PID=$!
     sleep 1
-    telnet localhost "$TELNET_TERMINAL_PORT"
+
+    if [[ $IS_TEST -eq 1 ]]; then
+        set +e
+        expect <<EOF
+        set timeout 1800
+        set retcode -1
+        spawn telnet localhost ${TELNET_TERMINAL_PORT}
+        expect -re {Test status: (-?\d+)} {
+            set retcode \$expect_out(1,string)
+        }
+        expect "Open IoT SDK unit-tests completed"
+        set retcode [expr -1*\$retcode]
+        exit \$retcode
+EOF
+        RETCODE=$?
+        FAILED_TESTS=$(expr "$FAILED_TESTS" + "$RETCODE")
+        echo "$(jq '. += {($testname): {failed: $result}}' --arg testname "$EXAMPLE" --arg result "$RETCODE" "$EXAMPLE_PATH"/test_report.json)" >"$EXAMPLE_PATH"/test_report.json
+    else
+        telnet localhost "$TELNET_TERMINAL_PORT"
+    fi
 
     # stop the fvp
     kill -9 "$FVP_PID" || true
+    set -e
     sleep 1
 }
 
@@ -246,7 +284,7 @@ if [[ $# -lt 1 ]]; then
 fi
 
 case "$1" in
-    shell | lock-app)
+    shell | unit-tests | lock-app)
         EXAMPLE=$1
         ;;
     *)
@@ -256,6 +294,29 @@ case "$1" in
         ;;
 esac
 
+if [[ "$EXAMPLE" == "unit-tests" ]]; then
+    if [ ! -z "$2" ]; then
+        if [[ " ${TEST_NAMES[*]} " =~ " $2 " ]]; then
+            if [[ "$COMMAND" != *"run"* ]]; then
+                echo "Test suites can only accept --command run"
+                show_usage
+                exit 2
+            fi
+            EXAMPLE=$2
+            echo "Run specific unit test $EXAMPLE"
+        elif [[ "$2" == "all" ]]; then
+            echo "Use all unit tests"
+        else
+            echo " Wrong unit test name"
+            show_usage
+            exit 2
+        fi
+    else
+        echo "Use all unit tests"
+    fi
+    IS_TEST=1
+fi
+
 case "$COMMAND" in
     build | run | test | build-run) ;;
     *)
@@ -266,7 +327,16 @@ case "$COMMAND" in
 esac
 
 TOOLCHAIN_PATH="toolchains/toolchain-$TOOLCHAIN.cmake"
-EXAMPLE_PATH="$CHIP_ROOT/examples/$EXAMPLE/openiotsdk"
+
+if [[ $IS_TEST -eq 0 ]]; then
+    EXAMPLE_PATH="$CHIP_ROOT/examples/$EXAMPLE/openiotsdk"
+else
+    EXAMPLE_PATH="$CHIP_ROOT/src/test_driver/openiotsdk/unit-tests"
+    if [[ -f $EXAMPLE_PATH/test_report.json ]]; then
+        rm -rf "$EXAMPLE_PATH"/test_report.json
+    fi
+    echo "{}" >"$EXAMPLE_PATH"/test_report.json
+fi
 
 if [ -z "$BUILD_PATH" ]; then
     BUILD_PATH="$EXAMPLE_PATH/build"
@@ -277,12 +347,35 @@ if [[ "$COMMAND" == *"build"* ]]; then
 fi
 
 if [[ "$COMMAND" == *"run"* ]]; then
-    run_fvp
+    # If user wants to run unit-tests we need to loop through all test names
+    if [[ "$EXAMPLE" == "unit-tests" ]]; then
+        if "$DEBUG"; then
+            echo "You have to specify the test suites to run in debug mode"
+            show_usage
+            exit 2
+        else
+            for NAME in "${TEST_NAMES[@]}"; do
+                EXAMPLE=$NAME
+                echo "$EXAMPLE_PATH"
+                echo "Run specific unit test $EXAMPLE"
+                run_fvp
+            done
+            echo "Failed tests total: $FAILED_TESTS"
+        fi
+    else
+        run_fvp
+    fi
 fi
 
 if [[ "$COMMAND" == *"test"* ]]; then
-    IS_TEST=1
-    run_test
+    if [[ "$EXAMPLE" == "unit-tests" ]]; then
+        echo "The test command can not be applied to the unit-tests example"
+        show_usage
+        exit 2
+    else
+        IS_TEST=1
+        run_test
+    fi
 fi
 
 if [[ $IS_TEST -eq 1 ]]; then