Skip to content

Commit

Permalink
Adding testbed implementation of CI tests;
Browse files Browse the repository at this point in the history
adding option to tests rvc4 against different versions of os;
  • Loading branch information
danilo-pejovic authored and Danilo committed Jan 31, 2025
1 parent 1643aae commit 9f5e229
Show file tree
Hide file tree
Showing 9 changed files with 206 additions and 56 deletions.
15 changes: 5 additions & 10 deletions .github/workflows/stability.workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,14 @@ jobs:

# Testing
test:
runs-on: ['self-hosted', 'hil-stability', 'linux']
runs-on: ['self-hosted', 'testbed-runner', 'linux']
timeout-minutes: 1450 # 24h & 10minutes
steps:
- name: Cache .hunter folder
uses: actions/cache@v3
with:
path: $HOME/.hun_vanilla
key: hunter-linux-stability-vanilla
- uses: actions/checkout@v3
with:
submodules: 'recursive'
- name: Prepare HIL Framework
run: source scripts/hil/prepare_hil_framework.sh

# TODO also modify above hunter key to 'asan'
# - name: Specify ASAN toolchain path
Expand All @@ -48,7 +45,5 @@ jobs:
# Release build
- name: Configure, Build and Test
run: |
cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun_vanilla -D DEPTHAI_BUILD_TESTS=ON
cmake --build build --parallel 8 --config Release --target stability_stress_test
cd build
../ci/stability_stress_test_combined.sh 86400
export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/$GITHUB_RUN_ID+stability"
exec hil --capabilities depthai-core-hil --reservation-name $RESERVATION_NAME -w -s --commands 'cd /tmp/depthai-core|| exit' 'scripts/hil/run_hil_stability.sh'
80 changes: 47 additions & 33 deletions .github/workflows/test.workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ concurrency:
jobs:

# Testing
test:
rvc2_test:
env:
VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite"
strategy:
Expand All @@ -33,14 +33,9 @@ jobs:
arch: 'x64'
flavor: 'tsan'
fail-fast: false
runs-on: ['self-hosted', 'hil-test-v3', '${{ matrix.os }}', '${{ matrix.arch }}']
runs-on: ['self-hosted', 'testbed-runner']

steps:
- name: Cache .hunter folder
uses: actions/cache@v3
with:
path: $HOME/.hun2_${{ matrix.flavor }}
key: hunter-${{ matrix.os }}-${{ matrix.cmake }}
- uses: actions/checkout@v3
with:
submodules: 'recursive'
Expand All @@ -51,37 +46,56 @@ jobs:
core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
- name: Create virtual environment
run: |
python -m venv venv
source venv/bin/activate
python -m pip install --upgrade pip
python -m pip install numpy
- name: Specify toolchain path
if: matrix.flavor == 'vanilla'
run: echo "CMAKE_TOOLCHAIN_PATH=" >> $GITHUB_ENV
- name: Specify toolchain path
if: matrix.flavor != 'vanilla'
run: echo "CMAKE_TOOLCHAIN_PATH=$PWD/cmake/toolchain/${{ matrix.flavor }}.cmake" >> $GITHUB_ENV
- name: Prepare HIL Framework
run: source scripts/hil/prepare_hil_framework.sh

- name: Configure, Build and Test
if: matrix.os != 'linux'
run: |
source venv/bin/activate # Activate virtual environment
cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun2_${{ matrix.flavor }} -D DEPTHAI_BUILD_EXAMPLES=ON -D DEPTHAI_BUILD_TESTS=ON -D DEPTHAI_TEST_EXAMPLES=ON -D DEPTHAI_BUILD_PYTHON=ON -D DEPTHAI_PYTHON_TEST_EXAMPLES=ON -D DEPTHAI_PYTHON_ENABLE_EXAMPLES=ON
cmake --build build --parallel 4 --config Release
cd tests
python3 run_tests.py
export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#rvc2-${{ matrix.os }}-${{ matrix.flavor }}"
exec hil --capabilities depthai-core-hil --reservation-name $RESERVATION_NAME --wait --sync-workspace --exclude-build --commands 'cd /tmp/depthai-core|| exit' 'scripts/hil/run_hil_tests.sh ${{ matrix.flavor }} --rvc2'
- name: Configure, Build and Test
if: matrix.os == 'linux'
run: |
export DISPLAY=:99
xdpyinfo -display $DISPLAY >/dev/null 2>&1 || (Xvfb $DISPLAY &)
source venv/bin/activate # Activate virtual environment
python3 -m pip install jinja2
cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun2_${{ matrix.flavor }} -D DEPTHAI_BUILD_EXAMPLES=ON -D DEPTHAI_BUILD_TESTS=ON -D DEPTHAI_TEST_EXAMPLES=ON -D DEPTHAI_BUILD_PYTHON=ON -D DEPTHAI_PYTHON_TEST_EXAMPLES=ON -D DEPTHAI_PYTHON_ENABLE_EXAMPLES=ON
cmake --build build --parallel 4 --config Release
cd tests
python3 run_tests.py
export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#rvc2-${{ matrix.os }}-${{ matrix.flavor }}"
exec hil --capabilities depthai-core-hil --reservation-name $RESERVATION_NAME --wait --sync-workspace --exclude-build --commands 'cd /tmp/depthai-core|| exit' 'scripts/hil/run_hil_tests.sh ${{ matrix.flavor }} --rvc2'
# Testing
rvc4_test:
env:
VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite"
strategy:
matrix:
# os: ['windows', 'macos', 'linux']
os: ['linux']
arch: ['x64']
flavor: ['vanilla', 'asan-ubsan', 'tsan']
rvc4os: ['r851.1.2']
exclude:
- os: 'windows'
arch: 'x64'
flavor: 'tsan'
fail-fast: false
runs-on: ['self-hosted', 'testbed-runner']

steps:
- uses: actions/checkout@v3
with:
submodules: 'recursive'

- name: Prepare HIL Framework
run: source scripts/hil/prepare_hil_framework.sh

- name: Configure, Build and Test
if: matrix.os != 'linux'
run: |
export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#rvc4-${{ matrix.os }}-${{ matrix.rvc4os }}-${{ matrix.flavor }}""
exec hil --capabilities depthai-core-hil --models oak4_pro --reservation-name $RESERVATION_NAME --wait --sync-workspace --exclude-build --commands 'cd /tmp/depthai-core|| exit' 'scripts/hil/run_hil_tests.sh ${{ matrix.flavor }} --rvc4 '
- name: Configure, Build and Test
if: matrix.os == 'linux'
run: |
export RESERVATION_NAME="https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID#rvc4-${{ matrix.os }}-${{ matrix.rvc4os }}-${{ matrix.flavor }}"
exec hil --capabilities depthai-core-hil --models oak4_pro --reservation-name $RESERVATION_NAME --wait --sync-workspace --exclude-build --rvc4-os-version ${{ matrix.rvc4os }} --commands 'cd /tmp/depthai-core|| exit' 'scripts/hil/run_hil_tests.sh ${{ matrix.flavor }} --rvc4'
31 changes: 31 additions & 0 deletions scripts/hil/powercycle_rvc2.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
from lib_testbed.power.PowerFactory import PowerFactory
from lib_testbed.config.Config import Config
from lib_testbed.utils.ssh_util import *
from time import sleep

testbed_name=os.getenv("SET_HIL_TESTBED")
config=Config(testbed_name)
print(config.cameras)

for camera in config.cameras:
if camera.platform == "rvc2":
print(camera.name)
power_control=PowerFactory.get_power_object_for_device(config, camera.name)
power_control[0].off()
sleep(2)
power_control[0].on()
for camera in config.cameras:
if camera.platform == "rvc4":
power_control=PowerFactory.get_power_object_for_device(config, camera.name)
if len(power_control) == 2:
power_control[1].off()
power_control[0].off()
sleep(2)
power_control[0].on()
power_control[1].on()
elif len(power_control) == 1:
power_control[0].off()
sleep(2)
power_control[0].on()
else:
print("No devices available in power_control.")
5 changes: 5 additions & 0 deletions scripts/hil/prepare_hil_framework.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/bash

echo "/home/$USER/hil_framework/lib_testbed/tools" >> $GITHUB_PATH
echo "PYTHONPATH="$PYTHONPATH:/home/$USER/hil_framework"" >> $GITHUB_ENV
echo "HIL_FRAMEWORK_PATH="/home/$USER/hil_framework"" >> $GITHUB_ENV
20 changes: 20 additions & 0 deletions scripts/hil/run_hil_stability.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
#!/bin/bash

# Set up a Python virtual environment
rm -rf venv
python3 -m venv venv
source venv/bin/activate
rm -rf build/
export PATH="$PATH:/home/hil/hil_framework/lib_testbed/tools"
export PYTHONPATH="$PYTHONPATH:/home/hil/hil_framework"
export HIL_FRAMEWORK_PATH="/home/hil/hil_framework"
source /home/hil/.SETUP_CONFIG_VARS

# Install required Python packages
pip install numpy pytest pytest-html > /dev/null 2>&1
pushd /home/$USER/hil_framework/ > /dev/null 2>&1 && pip install -r requirements.txt > /dev/null 2>&1 && popd > /dev/null 2>&1

cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun_vanilla -D DEPTHAI_BUILD_TESTS=ON
cmake --build build --parallel 8 --config Release --target stability_stress_test
cd build
../ci/stability_stress_test_combined.sh 86400
60 changes: 60 additions & 0 deletions scripts/hil/run_hil_tests.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/bin/bash

# Check if the argument is provided
if [ $# -lt 1 ]; then
echo "Usage: $0 <test_type> [--rvc2 | --rvc4]"
echo " test_type Specify test flavor: vanilla, asan-ubsan, or tsan."
echo " --rvc2 Optional: Run tests with RVC2 configuration."
echo " --rvc4 Optional: Run tests with RVC4 configuration."
exit 1
fi
# Get the test type from the argument
TEST_FLAVOR=$1
TEST_ARGS=$2

# Set up a Python virtual environment
rm -rf venv
python3 -m venv venv
source venv/bin/activate
export LC_ALL=en_US.UTF-8
locale

if [ "$TEST_FLAVOR" == "vanilla" ]; then
echo $CMAKE_TOOLCHAIN_PATH
else
export CMAKE_TOOLCHAIN_PATH=$PWD/cmake/toolchain/${FLAVOR}.cmake
fi

export PATH="$PATH:/home/hil/hil_framework/lib_testbed/tools"
export PYTHONPATH="$PYTHONPATH:/home/hil/hil_framework"
export HIL_FRAMEWORK_PATH="/home/hil/hil_framework"
source /home/hil/.SETUP_CONFIG_VARS

# Install required Python packages
pip install numpy pytest pytest-html > /dev/null 2>&1
pushd /home/$USER/hil_framework/ > /dev/null 2>&1 && git pull && git submodule update --init --recursive > /dev/null 2>&1 && popd > /dev/null 2>&1
pushd /home/$USER/hil_framework/ > /dev/null 2>&1 && pip install -r requirements.txt > /dev/null 2>&1 && popd > /dev/null 2>&1

# Check for optional RVC arguments
if [[ "$TEST_ARGS" == "--rvc4" ]]; then
echo "Running RVC4 configuration commands..."
adb root
adb shell systemctl stop agentconfd setup
adb shell systemctl disable agentconfd setup
adb shell mkdir -p /persist/factory
adb shell touch /persist/factory/enabled
adb shell reboot
echo "Device reboot initiated for RVC4. Factory mode enabled."
elif [[ "$TEST_ARGS" == "--rvc2" ]]; then
python scripts/hil/powercycle_rvc2.py
sleep 10
echo "RVC2 configuration specified. Continuing with test setup..."
fi

cmake -S . -B build -D CMAKE_BUILD_TYPE=Release -D HUNTER_ROOT=$HOME/.hun2_$TEST_FLAVOR -D DEPTHAI_BUILD_EXAMPLES=ON -D DEPTHAI_BUILD_TESTS=ON -D DEPTHAI_TEST_EXAMPLES=ON -D DEPTHAI_BUILD_PYTHON=ON -D DEPTHAI_PYTHON_TEST_EXAMPLES=ON -D DEPTHAI_PYTHON_ENABLE_EXAMPLES=ON
cmake --build build --parallel 2 --config Release

export DISPLAY=:99
xdpyinfo -display $DISPLAY >/dev/null 2>&1 || (Xvfb $DISPLAY &)
cd tests
python3 run_tests.py $TEST_ARGS
31 changes: 26 additions & 5 deletions tests/run_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,17 +77,29 @@ def compute_or_result(results):
default=default_path,
)

parser.add_argument(
"--rvc4",
action="store_true",
required=False,
)

parser.add_argument(
"--rvc2",
action="store_true",
required=False,
)

args = parser.parse_args()
test_dir = args.test_dir
print("Going to run tests in directory:", test_dir)
# cd to the test directory
os.chdir(pathlib.Path(test_dir).resolve())
# Example test configurations
test_configs = [
all_configs = [
{
"name": "Host",
"env": {},
"label": ["onhost"],
"labels": ["onhost"],
},
{
"name": "RVC4",
Expand All @@ -97,22 +109,31 @@ def compute_or_result(results):
{
"name": "RVC2 - USB",
"env": {"DEPTHAI_PLATFORM": "rvc2", "DEPTHAI_PROTOCOL": "usb"},
"label": ["rvc2", "usb"],
"labels": ["rvc2", "usb"],
},
{
"name": "RVC2 - POE",
"env": {"DEPTHAI_PLATFORM": "rvc2", "DEPTHAI_PROTOCOL": "tcpip"},
"label": ["rvc2", "poe"],
"labels": ["rvc2", "poe"],
},
]

# List to keep track of results
resultThreads = []

# Filter configurations based on command-line arguments
if args.rvc4==args.rvc2:
test_configs=all_configs
elif args.rvc4:
test_configs = [config for config in all_configs if "rvc4" in config.get("labels", []) or "onhost" in config.get("labels", [])]
elif args.rvc2:
test_configs = [config for config in all_configs if "rvc2" in config.get("labels", []) or "onhost" in config.get("labels", [])]


for config in test_configs:
name = config["name"]
env_vars = config["env"]
labels = config.get("labels") or config.get("label")
labels = config.get("labels")

print(f"Running tests for configuration: {name}")
resultThread = run_ctest(env_vars, labels, blocking=False, name=name)
Expand Down
13 changes: 7 additions & 6 deletions tests/src/ondevice_tests/device_usbspeed_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -70,12 +70,13 @@ TEST_CASE("Usb config modes") {
p.setBoardConfig(cfg.board);
}

SECTION("UsbSpeed::SUPER_PLUS") {
dai::DeviceBase::Config cfg;
cfg.board.usb.maxSpeed = dai::UsbSpeed::SUPER_PLUS;
speed = dai::UsbSpeed::SUPER_PLUS;
p.setBoardConfig(cfg.board);
}
// SECTION("UsbSpeed::SUPER_PLUS") {
// dai::DeviceBase::Config cfg;
// cfg.board.usb.maxSpeed = dai::UsbSpeed::SUPER_PLUS;
// speed = dai::UsbSpeed::SUPER_PLUS;
// p.setBoardConfig(cfg.board);
// }
//

dai::Device d(p);
REQUIRE(d.getUsbSpeed() == speed);
Expand Down
7 changes: 5 additions & 2 deletions tests/src/ondevice_tests/filesystem_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -246,11 +246,14 @@ TEST_CASE("dai::Path with DeviceBootloader") {
const std::wstring wstrBadfile(LPATH5);
const dai::Path diaBadWide(LPATH5);
#endif

std::this_thread::sleep_for(std::chrono::seconds(10));
bool found = false;
dai::DeviceInfo deviceInfo;
std::tie(found, deviceInfo) = dai::DeviceBootloader::getFirstAvailableDevice();
if(found) {
if(deviceInfo.state == X_LINK_BOOTLOADER) {
std::cout << "Device is already booted into bootloader mode. Booting tests will be skipped." << std::endl;
} else {
REQUIRE_NOTHROW([&]() {
dai::DeviceBootloader bl(deviceInfo);
auto currentBlType = bl.getType();
Expand Down Expand Up @@ -310,7 +313,7 @@ TEST_CASE("dai::Path with DeviceBootloader") {
}(),
ContainsSubstring("doesn't exist"));
#endif
} else {
}} else {
std::cout << "No devices found" << std::endl;
}
}
Expand Down

0 comments on commit 9f5e229

Please sign in to comment.