diff --git a/.drone.star b/.drone.star index 5301e22d9d8..be738ad8e9b 100644 --- a/.drone.star +++ b/.drone.star @@ -210,12 +210,11 @@ def gui_test_pipeline(ctx): ] # '--retry' and '--abortOnFail' are mutually exclusive - if "debug-gui-test" in ctx.build.title.lower() or "full-ci" in ctx.build.title.lower() or ctx.build.event in ("tag", "cron"): + if "full-ci" in ctx.build.title.lower() or ctx.build.event in ("tag", "cron"): # retry failed tests once squish_parameters.append("--retry 1") elif not "full-ci" in ctx.build.title.lower() and ctx.build.event == "pull_request": squish_parameters.append("--abortOnFail") - pass if params.get("skip", False): continue @@ -339,7 +338,7 @@ def gui_tests(squish_parameters = "", server_type = "oc10"): "STACKTRACE_FILE": "%s/stacktrace.log" % dir["guiTestReport"], "PLAYWRIGHT_BROWSERS_PATH": "%s/.playwright" % dir["base"], "OWNCLOUD_CORE_DUMP": 1, - "SCREEN_RECORD_ON_FAILURE": False, + "RECORD_VIDEO_ON_FAILURE": False, # allow to use any available pnpm version "COREPACK_ENABLE_STRICT": 0, }, diff --git a/test/gui/.pylintrc b/test/gui/.pylintrc index 146f973f5ed..11c27e4440c 100644 --- a/test/gui/.pylintrc +++ b/test/gui/.pylintrc @@ -18,6 +18,7 @@ ignore-paths=^tst_.*/test.py$, shared/scripts/custom_lib ignored-modules= squish, + squishinfo, object, objectmaphelper, test, diff --git a/test/gui/config.sample.ini b/test/gui/config.sample.ini index 1c374b19dc1..23dc09f61f7 100644 --- a/test/gui/config.sample.ini +++ b/test/gui/config.sample.ini @@ -11,4 +11,4 @@ TEMP_FOLDER_PATH= CLIENT_CONFIG_DIR= GUI_TEST_REPORT_DIR= OCIS=false -SCREEN_RECORD_ON_FAILURE=false \ No newline at end of file +RECORD_VIDEO_ON_FAILURE=false \ No newline at end of file diff --git a/test/gui/shared/scripts/bdd_hooks.py b/test/gui/shared/scripts/bdd_hooks.py index a5902b27772..992cdf4bb70 100644 --- a/test/gui/shared/scripts/bdd_hooks.py +++ b/test/gui/shared/scripts/bdd_hooks.py @@ -17,7 +17,6 @@ # manual for a complete reference of the available API. import shutil import os -import glob from urllib import request, error from datetime import datetime @@ -36,8 +35,9 @@ ) from helpers.api.utils import url_join from helpers.FilesHelper import prefix_path_namespace, cleanup_created_paths -from pageObjects.Toolbar import Toolbar +from helpers.ReportHelper import save_video_recording, take_screenshot +from pageObjects.Toolbar import Toolbar from pageObjects.AccountSetting import AccountSetting from pageObjects.AccountConnectionWizard import AccountConnectionWizard @@ -148,44 +148,9 @@ def scenario_failed(): ) -def get_screenshot_name(title): - return title.replace(" ", "_").replace("/", "_").strip(".") + ".png" - - -def get_screenrecord_name(title): - return title.replace(" ", "_").replace("/", "_").strip(".") + ".mp4" - - -def save_screenrecord(filename): - try: - # do not throw if stopVideoCapture() fails - test.stopVideoCapture() - except: - test.log("Failed to stop screen recording") - - if not (video_dir := squishinfo.resultDir): - video_dir = squishinfo.testCase - else: - test_case = "/".join(squishinfo.testCase.split("/")[-2:]) - video_dir = os.path.join(video_dir, test_case) - video_dir = os.path.join(video_dir, "attachments") - - if scenario_failed(): - video_files = glob.glob(f"{video_dir}/**/*.mp4", recursive=True) - screenrecords_dir = os.path.join( - get_config("guiTestReportDir"), "screenrecords" - ) - if not os.path.exists(screenrecords_dir): - os.makedirs(screenrecords_dir) - # reverse the list to get the latest video first - video_files.reverse() - for idx, video in enumerate(video_files): - if idx: - file_parts = filename.rsplit(".", 1) - filename = f"{file_parts[0]}_{idx+1}.{file_parts[1]}" - shutil.move(video, os.path.join(screenrecords_dir, filename)) - set_config("videoRecordCount", get_config("videoRecordCount") + 1) - shutil.rmtree(prefix_path_namespace(video_dir)) +def scenario_title_to_filename(title): + # scenario name can have "/" which is invalid filename + return title.replace(" ", "_").replace("/", "_").strip(".") # runs after every scenario @@ -195,26 +160,14 @@ def hook(context): clear_waited_after_sync() close_socket_connection() - # capture a screenshot if there is error or test failure in the current scenario execution - if scenario_failed() and os.getenv("CI") and is_linux(): - # scenario name can have "/" which is invalid filename - filename = get_screenshot_name(context.title) - directory = os.path.join(get_config("guiTestReportDir"), "screenshots") - if not os.path.exists(directory): - os.makedirs(directory) - try: - squish.saveDesktopScreenshot(os.path.join(directory, filename)) - except: - test.log("Failed to save screenshot") - - # check video report - if ( - get_config("screenRecordOnFailure") - or get_config("retrying") - and get_config("videoRecordCount") < get_config("videoRecordLimit") - ): - filename = get_screenrecord_name(context.title) - save_screenrecord(filename) + # generate screenshot and video reports + if is_linux(): + filename = scenario_title_to_filename(context.title) + if scenario_failed(): + take_screenshot(f"{filename}.png") + + if get_config("videoRecordingStarted"): + save_video_recording(f"{filename}.mp4", scenario_failed()) # teardown accounts and configs teardown_client() @@ -260,13 +213,6 @@ def hook(context): delete_created_users() -@OnFeatureEnd -def hook(context): - test.log("----------------") - test.log(str(get_config("videoRecordCount"))) - test.log("----------------") - - def teardown_client(): # Cleanup user accounts from UI for Windows platform # It is not needed for Linux so skipping it in order to save CI time diff --git a/test/gui/shared/scripts/helpers/ConfigHelper.py b/test/gui/shared/scripts/helpers/ConfigHelper.py index 407d3283c1c..83b09184665 100644 --- a/test/gui/shared/scripts/helpers/ConfigHelper.py +++ b/test/gui/shared/scripts/helpers/ConfigHelper.py @@ -73,7 +73,7 @@ def get_default_home_dir(): 'clientConfigDir': 'CLIENT_CONFIG_DIR', 'guiTestReportDir': 'GUI_TEST_REPORT_DIR', 'ocis': 'OCIS', - 'screenRecordOnFailure': 'SCREEN_RECORD_ON_FAILURE', + 'recordVideoOnFailure': 'RECORD_VIDEO_ON_FAILURE', } DEFAULT_PATH_CONFIG = { @@ -97,9 +97,9 @@ def get_default_home_dir(): 'clientConfigDir': get_config_home(), 'guiTestReportDir': os.path.abspath('../reports'), 'ocis': False, - 'screenRecordOnFailure': False, + 'recordVideoOnFailure': False, 'retrying': False, - 'videoRecordCount': 0, + 'videoRecordingStarted': False, } CONFIG.update(DEFAULT_PATH_CONFIG) @@ -114,7 +114,7 @@ def read_cfg_file(cfg_path): for key, _ in CONFIG.items(): if key in CONFIG_ENV_MAP: if value := cfg.get('DEFAULT', CONFIG_ENV_MAP[key]): - if key in ('ocis', 'screenRecordOnFailure'): + if key in ('ocis', 'recordVideoOnFailure'): CONFIG[key] = value == 'true' else: CONFIG[key] = value @@ -134,15 +134,13 @@ def init_config(): # read and override configs from environment variables for key, value in CONFIG_ENV_MAP.items(): if os.environ.get(value): - if key in ('ocis', 'screenRecordOnFailure'): + if key in ('ocis', 'recordVideoOnFailure'): CONFIG[key] = os.environ.get(value) == 'true' else: CONFIG[key] = os.environ.get(value) # Set the default values if empty for key, value in CONFIG.items(): - if key == 'videoRecordCount': - CONFIG[key] = get_config('videoRecordCount') if key in ('maxSyncTimeout', 'minSyncTimeout'): CONFIG[key] = builtins.int(value) elif key in ('localBackendUrl', 'middlewareUrl', 'secureLocalBackendUrl'): @@ -170,9 +168,7 @@ def set_config(key, value): if key in READONLY_CONFIG: raise KeyError(f'Cannot set read-only config: {key}') # save the initial config value - # 'videoRecordCount' is a special case, it is not a scenario config - # but is a global config throughout the test run - if key not in SCENARIO_CONFIGS and not key == 'videoRecordCount': + if key not in SCENARIO_CONFIGS: SCENARIO_CONFIGS[key] = CONFIG.get(key) CONFIG[key] = value diff --git a/test/gui/shared/scripts/helpers/ReportHelper.py b/test/gui/shared/scripts/helpers/ReportHelper.py new file mode 100644 index 00000000000..fe778105820 --- /dev/null +++ b/test/gui/shared/scripts/helpers/ReportHelper.py @@ -0,0 +1,75 @@ +import os +import glob +import shutil +import test +import squish +import squishinfo + +from helpers.ConfigHelper import get_config +from helpers.FilesHelper import prefix_path_namespace + + +def get_screenrecords_path(): + return os.path.join(get_config("guiTestReportDir"), "screenrecords") + + +def get_screenshots_path(): + return os.path.join(get_config("guiTestReportDir"), "screenshots") + + +def is_video_enabled(): + return ( + get_config("recordVideoOnFailure") + or get_config("retrying") + and not reached_video_limit() + ) + + +def reached_video_limit(): + video_report_dir = get_screenrecords_path() + if not os.path.exists(video_report_dir): + return False + entries = [f for f in os.scandir(video_report_dir) if f.is_file()] + return len(entries) >= get_config("videoRecordLimit") + + +def save_video_recording(filename, test_failed): + try: + # do not throw if stopVideoCapture() fails + test.stopVideoCapture() + except: + test.log("Failed to stop screen recording") + + if not (video_dir := squishinfo.resultDir): + video_dir = squishinfo.testCase + else: + test_case = "/".join(squishinfo.testCase.split("/")[-2:]) + video_dir = os.path.join(video_dir, test_case) + video_dir = os.path.join(video_dir, "attachments") + + # if the test failed + # move videos to the screenrecords directory + if test_failed: + video_files = glob.glob(f"{video_dir}/**/*.mp4", recursive=True) + screenrecords_dir = get_screenrecords_path() + if not os.path.exists(screenrecords_dir): + os.makedirs(screenrecords_dir) + # reverse the list to get the latest video first + video_files.reverse() + for idx, video in enumerate(video_files): + if idx: + file_parts = filename.rsplit(".", 1) + filename = f"{file_parts[0]}_{idx+1}.{file_parts[1]}" + shutil.move(video, os.path.join(screenrecords_dir, filename)) + # remove the video directory + shutil.rmtree(prefix_path_namespace(video_dir)) + + +def take_screenshot(filename): + directory = get_screenshots_path() + if not os.path.exists(directory): + os.makedirs(directory) + try: + squish.saveDesktopScreenshot(os.path.join(directory, filename)) + except: + test.log("Failed to save screenshot") diff --git a/test/gui/shared/scripts/helpers/SetupClientHelper.py b/test/gui/shared/scripts/helpers/SetupClientHelper.py index 337c8fa25e6..594b1ae6419 100644 --- a/test/gui/shared/scripts/helpers/SetupClientHelper.py +++ b/test/gui/shared/scripts/helpers/SetupClientHelper.py @@ -13,6 +13,7 @@ from helpers.SyncHelper import listen_sync_status_for_item from helpers.api.utils import url_join from helpers.UserHelper import get_displayname_for_user +from helpers.ReportHelper import is_video_enabled def substitute_inline_codes(value): @@ -103,12 +104,9 @@ def start_client(): + ' --logdebug' + ' --logflush' ) - if ( - get_config('screenRecordOnFailure') - or get_config('retrying') - and get_config('videoRecordCount') < get_config('videoRecordLimit') - ): + if is_video_enabled(): test.startVideoCapture() + set_config('videoRecordingStarted', True) def get_polling_interval():