[Python] Make it easier to run Python tests in CI (#15714)

* Run Python Test script on CI * Redirect log for pretty output * Update output * Fix * Update doc * Update script * Fix * Disable re-CASE * Update test * Add no-wifi variant * Fix log * Remove test in build.yaml * Enrich functions for enable and disable tests * Remove wait time, update doc * Fix * These words should not by typo?
project-chip · Apr 25, 2022 · 2251941 · 2251941
1 parent bd4da99
commit 2251941
Show file tree

Hide file tree

Showing 13 changed files with 708 additions and 127 deletions.
diff --git a/.github/.wordlist.txt b/.github/.wordlist.txt
@@ -1415,6 +1415,12 @@ kManage
 kOperate
 kView
 xFFFFFFFD
+ClusterObjectTests
+TestTimedRequestTimeout
+datamodel
+appliable
+commissionee
+configs
 NAMESERVER
 UTF
 localedef

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -265,6 +265,7 @@ jobs:
               run: |
                   scripts/run_in_build_env.sh 'pip3 install ./out/controller/python/chip-0.0-cp37-abi3-linux_x86_64.whl'
                   scripts/run_in_build_env.sh '(cd src/controller/python/test/unit_tests/ && python3 -m unittest -v)'
+
     build_darwin:
         name: Build on Darwin (clang, python_lib, simulated)
         timeout-minutes: 200

diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -216,3 +216,169 @@ jobs:
                   path: objdir-clone/
                   # objdirs are big; don't hold on to them too long.
                   retention-days: 5
+    repl_tests_linux:
+        name: REPL Tests - Linux
+        timeout-minutes: 120
+
+        strategy:
+            matrix:
+                build_variant: [no-ble-no-wifi-tsan]
+
+        env:
+            BUILD_VARIANT: ${{matrix.build_variant}}
+            TSAN_OPTIONS: "halt_on_error=1 suppressions=scripts/tests/chiptest/tsan-linux-suppressions.txt"
+
+        if: github.actor != 'restyled-io[bot]'
+        runs-on: ubuntu-latest
+
+        container:
+            image: connectedhomeip/chip-build:0.5.56
+            options:
+                --privileged --sysctl "net.ipv6.conf.all.disable_ipv6=0
+                net.ipv4.conf.all.forwarding=1 net.ipv6.conf.all.forwarding=1"
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v2
+              with:
+                  submodules: true
+            - name:
+                  Try to ensure the directories for core dumping exist and we
+                  can write them.
+              run: |
+                  mkdir /tmp/cores || true
+                  sysctl -w kernel.core_pattern=/tmp/cores/core.%u.%p.%t || true
+                  mkdir objdir-clone || true
+            - name: Bootstrap
+              timeout-minutes: 10
+              run: scripts/build/gn_bootstrap.sh
+            - name: Uploading bootstrap logs
+              uses: actions/upload-artifact@v2
+              if: ${{ always() }} && ${{ !env.ACT }}
+              with:
+                  name:
+                      bootstrap-logs-linux-${{ matrix.build_variant }}${{ matrix.chip_tool }}
+                  path: |
+                      .environment/gn_out/.ninja_log
+                      .environment/pigweed-venv/*.log
+            - name: Build Python REPL and example apps
+              timeout-minutes: 50
+              run: |
+                    scripts/run_in_build_env.sh './scripts/build_python.sh --install_wheel build-env'
+                    ./scripts/run_in_build_env.sh \
+                     "./scripts/build/build_examples.py \
+                        --target linux-x64-all-clusters-${BUILD_VARIANT} \
+                        build \
+                        --copy-artifacts-to objdir-clone \
+                     "
+            - name: Run Tests
+              timeout-minutes: 30
+              run: |
+                    scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app chip-all-clusters-app --factoryreset -- -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout'
+            - name: Uploading core files
+              uses: actions/upload-artifact@v2
+              if: ${{ failure() }} && ${{ !env.ACT }}
+              with:
+                  name:
+                      crash-core-linux-python-repl
+                  path: /tmp/cores/
+                  # Cores are big; don't hold on to them too long.
+                  retention-days: 5
+            - name: Uploading objdir for debugging
+              uses: actions/upload-artifact@v2
+              if: ${{ failure() }} && ${{ !env.ACT }}
+              with:
+                  name:
+                      crash-objdir-linux-python-repl
+                  path: objdir-clone/
+                  # objdirs are big; don't hold on to them too long.
+                  retention-days: 5
+
+    repl_tests_darwin:
+        name: REPL Tests - Darwin
+        timeout-minutes: 120
+
+        strategy:
+            matrix:
+                build_variant: [no-ble-no-wifi-tsan]
+        env:
+            BUILD_VARIANT: ${{matrix.build_variant}}
+            TSAN_OPTIONS: "halt_on_error=1"
+
+        if: github.actor != 'restyled-io[bot]'
+        runs-on: macos-latest
+
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v2
+              with:
+                  submodules: true
+            - name: Setup Environment
+              # coreutils for stdbuf
+              run: brew install openssl pkg-config coreutils
+            - name:
+                  Try to ensure the directories for core dumping and diagnostic
+                  log collection exist and we can write them.
+              run: |
+                  sudo chown ${USER} /cores || true
+                  mkdir -p ~/Library/Logs/DiagnosticReports || true
+                  mkdir objdir-clone || true
+            - name: Fix pkgconfig link
+              working-directory: /usr/local/lib/pkgconfig
+              run: |
+                  pwd
+                  ls -la /usr/local/Cellar/
+                  ls -la /usr/local/Cellar/openssl@1.1
+                  OPEN_SSL_VERSION=`ls -la /usr/local/Cellar/openssl@1.1 | cat | tail -n1 | awk '{print $NF}'`
+                  ln -s /usr/local/Cellar/openssl@1.1/$OPEN_SSL_VERSION/lib/pkgconfig/* .
+            - name: Bootstrap
+              timeout-minutes: 25
+              run: scripts/build/gn_bootstrap.sh
+            - name: Uploading bootstrap logs
+              uses: actions/upload-artifact@v2
+              if: ${{ always() }} && ${{ !env.ACT }}
+              with:
+                  name:
+                      bootstrap-logs-darwin-${{ matrix.build_variant }}${{ matrix.chip_tool }}
+                  path: |
+                      .environment/gn_out/.ninja_log
+                      .environment/pigweed-venv/*.log
+            - name: Build Python REPL and example apps
+              timeout-minutes: 50
+              run: |
+                    scripts/run_in_build_env.sh './scripts/build_python.sh --install_wheel build-env'
+                    ./scripts/run_in_build_env.sh \
+                     "./scripts/build/build_examples.py \
+                        --target darwin-x64-all-clusters-${BUILD_VARIANT} \
+                        build \
+                        --copy-artifacts-to objdir-clone \
+                     "
+            - name: Run Tests
+              timeout-minutes: 30
+              run: |
+                    scripts/run_in_build_env.sh './scripts/tests/run_python_test.py --app chip-all-clusters-app --factoryreset --app-params "--discriminator 3840 --interface-id -1" -- -t 3600 --disable-test ClusterObjectTests.TestTimedRequestTimeout'
+            - name: Uploading core files
+              uses: actions/upload-artifact@v2
+              if: ${{ failure() }} && ${{ !env.ACT }}
+              with:
+                  name:
+                      crash-core-darwin-python-repl
+                  path: /cores/
+                  # Cores are big; don't hold on to them too long.
+                  retention-days: 5
+            - name: Uploading diagnostic logs
+              uses: actions/upload-artifact@v2
+              if: ${{ failure() }} && ${{ !env.ACT }}
+              with:
+                  name:
+                      crash-log-darwin-python-repl
+                  path: ~/Library/Logs/DiagnosticReports/
+            - name: Uploading objdir for debugging
+              uses: actions/upload-artifact@v2
+              if: ${{ failure() }} && ${{ !env.ACT }}
+              with:
+                  name:
+                      crash-objdir-darwin-python-repl
+                  path: objdir-clone/
+                  # objdirs are big; don't hold on to them too long.
+                  retention-days: 5
diff --git a/docs/guides/matter-repl.md b/docs/guides/matter-repl.md
@@ -163,3 +163,85 @@ launched into the playground:
 [Multi Fabric Commissioning](https://deepnote.com/viewer/github/project-chip/connectedhomeip/blob/master/docs/guides/repl/Matter%20-%20Multi%20Fabric%20Commissioning.ipynb)
 
 [Access Control](https://deepnote.com/viewer/github/project-chip/connectedhomeip/blob/master/docs/guides/repl/Matter%20-%20Access%20Control.ipynb)
+
+## Testing
+
+We also provide `mobile-device-test.py` for testing your accessories, you can
+run it manually or using a wrapper script.
+
+### Usage
+
+mobile-device-test.py provides the following options for running the tests:
+
+```
+  --controller-nodeid INTEGER     NodeId of the controller.
+  --device-nodeid INTEGER         NodeId of the device.
+  -a, --address TEXT              Skip commissionee discovery, commission the
+                                  device with the IP directly.
+
+  -t, --timeout INTEGER           The program will return with timeout after
+                                  specified seconds.
+
+  --discriminator INTEGER         Discriminator of the device.
+  --setup-pin INTEGER             Setup pincode of the device.
+  --enable-test TEXT              The tests to be executed. By default, all
+                                  tests will be executed, use this option to
+                                  run a specific set of tests. Use --print-
+                                  test-list for a list of appliable tests.
+
+  --disable-test TEXT             The tests to be excluded from the set of
+                                  enabled tests. Use --print-test-list for a
+                                  list of appliable tests.
+
+  --log-level [ERROR|WARN|INFO|DEBUG]
+                                  The log level of the test.
+  --log-format TEXT               Override logging format
+  --print-test-list               Print a list of test cases and test sets
+                                  that can be toggled via --enable-test and
+                                  --disable-test, then exit
+
+  --help                          Show this message and exit.
+```
+
+By default, all tests will be executed, however, you can exclude one or more
+tests or only include a few tests if you want.
+
+For example, if you are working for commissioning, then you may want to exclude
+the data model test cases by adding `--disable-test datamodel` to disable all
+data model tests.
+
+Some tests provides the option to exclude them. For example, you can use
+`--disable-test ClusterObjectTests.TestTimedRequestTimeout` to exclude the
+"TestTimedRequestTimeout" test case.
+
+It is recommanded to use the test wrapper to run mobile-device-test.py, for
+example, you can run:
+
+```
+./scripts/tests/run_python_test.py --app chip-all-clusters-app --factoryreset
+```
+
+It provides some extra options, for example:
+
+```
+  --app TEXT         Local application to use, omit to use external apps, use
+                     a path for a specific binary or use a filename to search
+                     under the current matter checkout.
+
+  --factoryreset     Remove app config and repl configs (/tmp/chip* and
+                     /tmp/repl*) before running the tests.
+
+  --app-params TEXT  The extra parameters passed to the device.
+  --script PATH      Test script to use.
+  --help             Show this message and exit.
+```
+
+You can pass your own flags for mobile-device-test.py by appending them to the
+command line with two dashes, for example:
+
+```
+./scripts/tests/run_python_test.py --app chip-all-clusters-app --factoryreset -- -t 90 --disable-test ClusterObjectTests.TestTimedRequestTimeout
+```
+
+will pass `-t 90 --disable-test ClusterObjectTests.TestTimedRequestTimeout` to
+`mobile-device-test.py`
diff --git a/scripts/build/build/targets.py b/scripts/build/build/targets.py
@@ -229,6 +229,7 @@ def HostTargets():
     # builds is exponential here
     builder.AppendVariant(name="ipv6only", enable_ipv4=False),
     builder.AppendVariant(name="no-ble", enable_ble=False),
+    builder.AppendVariant(name="no-wifi", enable_wifi=False),
     builder.AppendVariant(name="tsan", conflicts=['asan'], use_tsan=True),
     builder.AppendVariant(name="asan", conflicts=['tsan'], use_asan=True),
     builder.AppendVariant(

diff --git a/scripts/build/builders/host.py b/scripts/build/builders/host.py
@@ -154,7 +154,7 @@ def PlatformName(self):
 class HostBuilder(GnBuilder):
 
     def __init__(self, root, runner, app: HostApp, board=HostBoard.NATIVE, enable_ipv4=True,
-                 enable_ble=True, use_tsan=False,  use_asan=False, separate_event_loop=True,
+                 enable_ble=True, enable_wifi=True, use_tsan=False,  use_asan=False, separate_event_loop=True,
                  test_group=False, use_libfuzzer=False, use_clang=False,
                  use_platform_mdns=False):
         super(HostBuilder, self).__init__(
@@ -171,6 +171,9 @@ def __init__(self, root, runner, app: HostApp, board=HostBoard.NATIVE, enable_ip
         if not enable_ble:
             self.extra_gn_options.append('chip_config_network_layer_ble=false')
 
+        if not enable_wifi:
+            self.extra_gn_options.append('chip_enable_wifi=false')
+
         if use_tsan:
             self.extra_gn_options.append('is_tsan=true')
 

diff --git a/scripts/build_python.sh b/scripts/build_python.sh
@@ -42,6 +42,7 @@ declare chip_detail_logging=false
 declare enable_pybindings=false
 declare chip_mdns
 declare case_retry_delta
+declare install_wheel=no
 
 help() {
 
@@ -58,6 +59,10 @@ Input Options:
 
   -t --time_between_case_retries MRPActiveRetryInterval     Specify MRPActiveRetryInterval value
                                                             Default is 300 ms
+  -i, --install_wheel no|build-env|separate                 Where to install the Python wheel
+                                                            no: Do not install
+                                                            build-env: install to virtual env for build matter
+                                                            separate: install to another virtual env (out/python_env)
 "
 }
 
@@ -85,6 +90,10 @@ while (($#)); do
             chip_case_retry_delta=$2
             shift
             ;;
+        --install_wheel | -i)
+            install_wheel=$2
+            shift
+            ;;
         -*)
             help
             echo "Unknown Option \"$1\""
@@ -114,24 +123,34 @@ else
     ninja -C "$OUTPUT_ROOT" python
 fi
 
-# Create a virtual environment that has access to the built python tools
-virtualenv --clear "$ENVIRONMENT_ROOT"
-
-# Activate the new environment to register the python WHL
-
 if [ "$enable_pybindings" == true ]; then
     WHEEL=$(ls "$OUTPUT_ROOT"/pybindings/pycontroller/pychip-*.whl | head -n 1)
 else
     WHEEL=$(ls "$OUTPUT_ROOT"/controller/python/chip-*.whl | head -n 1)
 fi
 
-source "$ENVIRONMENT_ROOT"/bin/activate
-"$ENVIRONMENT_ROOT"/bin/python -m pip install --upgrade pip
-"$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall --no-cache-dir "$WHEEL"
-
-echo ""
-echo_green "Compilation completed and WHL package installed in: "
-echo_blue "  $ENVIRONMENT_ROOT"
-echo ""
-echo_green "To use please run:"
-echo_bold_white "  source $ENVIRONMENT_ROOT/bin/activate"
+if [ "$install_wheel" = "no" ]; then
+    exit 0
+elif [ "$install_wheel" = "separate" ]; then
+    # Create a virtual environment that has access to the built python tools
+    virtualenv --clear "$ENVIRONMENT_ROOT"
+
+    source "$ENVIRONMENT_ROOT"/bin/activate
+    "$ENVIRONMENT_ROOT"/bin/python -m pip install --upgrade pip
+    "$ENVIRONMENT_ROOT"/bin/pip install --upgrade --force-reinstall --no-cache-dir "$WHEEL"
+
+    echo ""
+    echo_green "Compilation completed and WHL package installed in: "
+    echo_blue "  $ENVIRONMENT_ROOT"
+    echo ""
+    echo_green "To use please run:"
+    echo_bold_white "  source $ENVIRONMENT_ROOT/bin/activate"
+elif [ "$install_wheel" = "build-env" ]; then
+    pip install --force-reinstall "$WHEEL"
+
+    echo ""
+    echo_green "Compilation completed and WHL package installed in virtualenv for building sdk"
+    echo ""
+    echo_green "To use please run:"
+    echo_bold_white "  source $CHIP_ROOT/scripts/activate.sh"
+fi