From 2a0c4ead4f46d5d4fbc5ac3c9f849c862bb87b46 Mon Sep 17 00:00:00 2001 From: Dean Brettle Date: Sat, 31 Aug 2024 06:41:08 -0500 Subject: [PATCH] Fix-test-suite-not-run-on-macos (#6631) Fixes to run test_suite.yml during macos CI (if "test suite mac" label is present). * Make sure the Contents/Frameworks folder is created. * Set WEBOTS_HOME_PATH such that tests can find Makefile.include during CI. * Change WEBOTS_HOME to Webots.app and add logic to test_suite.py to find webots executable. * Get qt_warning_filters.conf from correct location on macos. * Ignore (but warn about) error trying to connect to background Webots on MacOS. * On Mac, timeout test after 60 minutes instead of 10. * Use macos-13 runner instead of macos-14. The former has twice as much RAM as the latter. * Install urdfdom during macos CI because it is needed by tests suite. * Assert that move succeeds. * Print age of ode_tmp.dif so we can see how long we should wait on macos during CI. * Allow up to 20 seconds for world reload. * Skip billboard.wbt test during macos CI. The test passes locally, so the failure is presumably due to the CI server being headless. * Only increase timeout on macos. --------- Co-authored-by: CoolSpy3 <55305038+CoolSpy3@users.noreply.github.com> Co-authored-by: Olivier Michel --- .github/workflows/test_suite_mac.yml | 20 +++++++++++-------- src/webots/gui/main.cpp | 6 +++++- .../test_suite_supervisor.py | 3 +++ .../controllers/determinism/determinism.c | 6 +++--- tests/test_suite.py | 12 +++++++++-- 5 files changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test_suite_mac.yml b/.github/workflows/test_suite_mac.yml index aa526a8d088..90206863f68 100644 --- a/.github/workflows/test_suite_mac.yml +++ b/.github/workflows/test_suite_mac.yml @@ -19,7 +19,7 @@ jobs: if: ${{ contains(github.event.pull_request.labels.*.name, 'test webots build') }} strategy: matrix: - os: [macos-14] + os: [macos-13] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -33,7 +33,7 @@ jobs: if: ${{ github.event_name == 'push' || github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'test distribution') || contains(github.event.pull_request.labels.*.name, 'test suite') }} strategy: matrix: - os: [macos-14] + os: [macos-13] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -67,10 +67,10 @@ jobs: distribution/*.dmg test-suite: needs: build - if: ${{ contains(github.event.pull_request.labels.*.name, 'test suite') }} + if: ${{ contains(github.event.pull_request.labels.*.name, 'test suite mac') }} strategy: matrix: - os: [macos-14] + os: [macos-13] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -89,17 +89,21 @@ jobs: with: python-version: 3.11 - name: Test - run: + run: | + brew install urdfdom export WEBOTS_DISABLE_SAVE_SCREEN_PERSPECTIVE_ON_CLOSE=true + python scripts/packaging/update_urls.py $(git log -1 --format='%H') export WEBOTS_HOME=/Applications/Webots.app - # we currently can't run the test suite because of https://github.com/cyberbotics/webots/issues/2110 - # python tests/test_suite.py + export TESTS_HOME=$PWD # required by cache group in the test suite + export WEBOTS_HOME_PATH=$PWD # required for tests to find Makefile.include + export BRANCH_HASH=$(git log -1 --format='%H') + python tests/test_suite.py delete-artifacts: needs: [build, test-suite] if: ${{ always() && !contains(github.event.pull_request.labels.*.name, 'test distribution') && !contains(github.event.pull_request.labels.*.name, 'test webots build') }} strategy: matrix: - os: [macos-14] + os: [macos-13] runs-on: ubuntu-latest steps: - name: Delete artifacts diff --git a/src/webots/gui/main.cpp b/src/webots/gui/main.cpp index 84a8a6d736c..2b49b9189b8 100644 --- a/src/webots/gui/main.cpp +++ b/src/webots/gui/main.cpp @@ -147,8 +147,12 @@ int main(int argc, char *argv[]) { #endif ); - // load qt warning filters from file +#ifdef __APPLE__ + QString qtFiltersFilePath = QDir::fromNativeSeparators(webotsDirPath + "/Contents/Resources/qt_warning_filters.conf"); +#else QString qtFiltersFilePath = QDir::fromNativeSeparators(webotsDirPath + "/resources/qt_warning_filters.conf"); +#endif + // load qt warning filters from file QFile qtFiltersFile(qtFiltersFilePath); if (qtFiltersFile.open(QIODevice::ReadOnly)) { QTextStream in(&qtFiltersFile); diff --git a/tests/default/controllers/test_suite_supervisor/test_suite_supervisor.py b/tests/default/controllers/test_suite_supervisor/test_suite_supervisor.py index fafbd52126f..59889494f0f 100644 --- a/tests/default/controllers/test_suite_supervisor/test_suite_supervisor.py +++ b/tests/default/controllers/test_suite_supervisor/test_suite_supervisor.py @@ -240,6 +240,9 @@ def run(self): # 30 seconds before executing the next world, 60 seconds for the robot_window_html test delay = 60 if self.currentSimulationFilename.endswith('/robot_window_html.wbt') else 30 + if sys.platform == "darwin": + # Longer timeout on MacOS because Webots takes longer to start there during CI. + delay *= 10 timeout = time.time() + delay running_controllers_pid = [] diff --git a/tests/physics/controllers/determinism/determinism.c b/tests/physics/controllers/determinism/determinism.c index 764953a2116..e91ac27cb1e 100644 --- a/tests/physics/controllers/determinism/determinism.c +++ b/tests/physics/controllers/determinism/determinism.c @@ -28,8 +28,8 @@ int main(int argc, char **argv) { if (file_exists(ode_tmp_dif)) { double delta = difftime(now, file_get_creation_time(ode_tmp_dif)); - // printf("delta time = %f\n", delta); - if (delta > 2.0) { + // printf("%s is %f seconds old.\n", ode_tmp_dif, delta); + if (delta > 20.0) { ts_assert_boolean_equal(remove_file(ode_tmp_dif), "Cannot remove ODE tmp dif file"); ts_assert_boolean_not_equal(file_exists(ode_tmp_dif), "Cannot remove ODE tmp dif file"); } @@ -60,7 +60,7 @@ int main(int argc, char **argv) { exit(EXIT_FAILURE); } } else { - move_file(ode_dif, ode_tmp_dif); + ts_assert_boolean_equal(move_file(ode_dif, ode_tmp_dif), "Cannot move ODF dif file to ODE tmp dif file"); wb_supervisor_world_reload(); wb_robot_cleanup(); exit(EXIT_SUCCESS); diff --git a/tests/test_suite.py b/tests/test_suite.py index 2fff35b42e9..dac4d398c09 100755 --- a/tests/test_suite.py +++ b/tests/test_suite.py @@ -86,6 +86,8 @@ def setupWebots(): if sys.platform == 'win32': webotsFullPath = os.path.join(os.path.normpath(os.environ['WEBOTS_HOME']), 'msys64', 'mingw64', 'bin', 'webots.exe') + elif sys.platform == 'darwin': + webotsFullPath = os.path.join(os.path.normpath(os.environ['WEBOTS_HOME']), 'Contents', 'MacOS', 'webots') else: webotsBinary = 'webots' if 'WEBOTS_HOME' in os.environ: @@ -198,12 +200,14 @@ def generateWorldsList(groupName): for filename in filenames: # speaker test not working on github action because of missing sound drivers # robot window and movie recording test not working on BETA Ubuntu 22.04 GitHub Action environment + # billboard test not working in macos GitHub Action environment if (not filename.endswith('_temp.wbt') and not ('GITHUB_ACTIONS' in os.environ and ( filename.endswith('speaker.wbt') or filename.endswith('local_proto_with_texture.wbt') or (filename.endswith('robot_window_html.wbt') and is_ubuntu_22_04) or - (filename.endswith('supervisor_start_stop_movie.wbt') and is_ubuntu_22_04) + (filename.endswith('supervisor_start_stop_movie.wbt') and is_ubuntu_22_04) or + (filename.endswith('billboard.wbt') and sys.platform == 'darwin') ))): worldsList.append(filename) @@ -244,7 +248,11 @@ def runGroupTest(groupName, firstSimulation, worldsCount, failures): command = Command(webotsArguments) # redirect stdout and stderr to files - command.runTest(timeout=10 * 60) # 10 minutes + timeoutMinutes = 10 + if sys.platform == "darwin": + # Longer timeout on MacOS because Webots takes longer to start there during CI. + timeoutMinutes = 60 + command.runTest(timeout=timeoutMinutes * 60) if command.isTimeout or command.returncode != 0: if command.isTimeout: