[gui-tests][full-ci] retry failed scenario (#11920)

* test: retry failed scenario

* test: report helper

* tests: use snake_case

.drone.star

@@ -209,7 +209,11 @@ def gui_test_pipeline(ctx):
             "--tags ~@skipOnLinux",
         ]
 
-        if not "full-ci" in ctx.build.title.lower() and ctx.build.event == "pull_request":
+        # '--retry' and '--abortOnFail' are mutually exclusive
+        if "full-ci" in ctx.build.title.lower() or ctx.build.event in ("tag", "cron"):
+            # retry failed tests once
+            squish_parameters.append("--retry 1")
+        elif not "full-ci" in ctx.build.title.lower() and ctx.build.event == "pull_request":
             squish_parameters.append("--abortOnFail")
 
         if params.get("skip", False):
@@ -334,7 +338,7 @@ def gui_tests(squish_parameters = "", server_type = "oc10"):
             "STACKTRACE_FILE": "%s/stacktrace.log" % dir["guiTestReport"],
             "PLAYWRIGHT_BROWSERS_PATH": "%s/.playwright" % dir["base"],
             "OWNCLOUD_CORE_DUMP": 1,
-            "SCREEN_RECORD_ON_FAILURE": False,
+            "RECORD_VIDEO_ON_FAILURE": False,
             # allow to use any available pnpm version
             "COREPACK_ENABLE_STRICT": 0,
         },

test/gui/.pylintrc

@@ -18,6 +18,7 @@ ignore-paths=^tst_.*/test.py$,
     shared/scripts/custom_lib
 ignored-modules=
     squish,
+    squishinfo,
     object,
     objectmaphelper,
     test,

test/gui/config.sample.ini

@@ -11,4 +11,4 @@ TEMP_FOLDER_PATH=
 CLIENT_CONFIG_DIR=
 GUI_TEST_REPORT_DIR=
 OCIS=false
-SCREEN_RECORD_ON_FAILURE=false
+RECORD_VIDEO_ON_FAILURE=false

test/gui/shared/scripts/bdd_hooks.py

@@ -17,7 +17,6 @@
 # manual for a complete reference of the available API.
 import shutil
 import os
-import glob
 from urllib import request, error
 from datetime import datetime
 
@@ -36,8 +35,9 @@
 )
 from helpers.api.utils import url_join
 from helpers.FilesHelper import prefix_path_namespace, cleanup_created_paths
-from pageObjects.Toolbar import Toolbar
+from helpers.ReportHelper import save_video_recording, take_screenshot
 
+from pageObjects.Toolbar import Toolbar
 from pageObjects.AccountSetting import AccountSetting
 from pageObjects.AccountConnectionWizard import AccountConnectionWizard
 
@@ -50,6 +50,7 @@
 # this will reset in every test suite
 PREVIOUS_FAIL_RESULT_COUNT = 0
 PREVIOUS_ERROR_RESULT_COUNT = 0
+PREVIOUS_SCENARIO = ""
 
 
 # runs before a feature
@@ -65,6 +66,11 @@ def hook(context):
 def hook(context):
     unlock_keyring()
     clear_scenario_config()
+    global PREVIOUS_SCENARIO
+    if PREVIOUS_SCENARIO == context.title:
+        test.log("[INFO] Retrying this failed scenario...")
+        set_config("retrying", True)
+    PREVIOUS_SCENARIO = context.title
 
 
 # runs before every scenario
@@ -142,44 +148,9 @@ def scenario_failed():
     )
 
 
-def get_screenshot_name(title):
-    return title.replace(" ", "_").replace("/", "_").strip(".") + ".png"
-
-
-def get_screenrecord_name(title):
-    return title.replace(" ", "_").replace("/", "_").strip(".") + ".mp4"
-
-
-def save_screenrecord(filename):
-    try:
-        # do not throw if stopVideoCapture() fails
-        test.stopVideoCapture()
-    except:
-        test.log("Failed to stop screen recording")
-
-    if not (video_dir := squishinfo.resultDir):
-        video_dir = squishinfo.testCase
-    else:
-        test_case = "/".join(squishinfo.testCase.split("/")[-2:])
-        video_dir = os.path.join(video_dir, test_case)
-    video_dir = os.path.join(video_dir, "attachments")
-
-    if scenario_failed():
-        video_files = glob.glob(f"{video_dir}/**/*.mp4", recursive=True)
-        screenrecords_dir = os.path.join(
-            get_config("guiTestReportDir"), "screenrecords"
-        )
-        if not os.path.exists(screenrecords_dir):
-            os.makedirs(screenrecords_dir)
-        # reverse the list to get the latest video first
-        video_files.reverse()
-        for idx, video in enumerate(video_files):
-            if idx:
-                file_parts = filename.rsplit(".", 1)
-                filename = f"{file_parts[0]}_{idx+1}.{file_parts[1]}"
-            shutil.move(video, os.path.join(screenrecords_dir, filename))
-
-    shutil.rmtree(prefix_path_namespace(video_dir))
+def scenario_title_to_filename(title):
+    # scenario name can have "/" which is invalid filename
+    return title.replace(" ", "_").replace("/", "_").strip(".")
 
 
 # runs after every scenario
@@ -189,22 +160,14 @@ def hook(context):
     clear_waited_after_sync()
     close_socket_connection()
 
-    # capture a screenshot if there is error or test failure in the current scenario execution
-    if scenario_failed() and os.getenv("CI") and is_linux():
-        # scenario name can have "/" which is invalid filename
-        filename = get_screenshot_name(context.title)
-        directory = os.path.join(get_config("guiTestReportDir"), "screenshots")
-        if not os.path.exists(directory):
-            os.makedirs(directory)
-        try:
-            squish.saveDesktopScreenshot(os.path.join(directory, filename))
-        except:
-            test.log("Failed to save screenshot")
-
-    # check video report
-    if get_config("screenRecordOnFailure"):
-        filename = get_screenrecord_name(context.title)
-        save_screenrecord(filename)
+    # generate screenshot and video reports
+    if is_linux():
+        filename = scenario_title_to_filename(context.title)
+        if scenario_failed():
+            take_screenshot(f"{filename}.png")
+
+        if get_config("video_recording_started"):
+            save_video_recording(f"{filename}.mp4", scenario_failed())
 
     # teardown accounts and configs
     teardown_client()

test/gui/shared/scripts/helpers/ConfigHelper.py

@@ -73,12 +73,14 @@ def get_default_home_dir():
     'clientConfigDir': 'CLIENT_CONFIG_DIR',
     'guiTestReportDir': 'GUI_TEST_REPORT_DIR',
     'ocis': 'OCIS',
-    'screenRecordOnFailure': 'SCREEN_RECORD_ON_FAILURE',
+    'record_video_on_failure': 'RECORD_VIDEO_ON_FAILURE',
 }
 
 DEFAULT_PATH_CONFIG = {
     'custom_lib': os.path.abspath('../shared/scripts/custom_lib'),
     'home_dir': get_default_home_dir(),
+    # allow to record first 5 videos
+    'video_record_limit': 5,
 }
 
 # default config values
@@ -95,20 +97,24 @@ def get_default_home_dir():
     'clientConfigDir': get_config_home(),
     'guiTestReportDir': os.path.abspath('../reports'),
     'ocis': False,
-    'screenRecordOnFailure': False,
+    'record_video_on_failure': False,
+    'retrying': False,
+    'video_recording_started': False,
 }
 CONFIG.update(DEFAULT_PATH_CONFIG)
 
 READONLY_CONFIG = list(CONFIG_ENV_MAP.keys()) + list(DEFAULT_PATH_CONFIG.keys())
 
+SCENARIO_CONFIGS = {}
+
 
 def read_cfg_file(cfg_path):
     cfg = ConfigParser()
     if cfg.read(cfg_path):
         for key, _ in CONFIG.items():
             if key in CONFIG_ENV_MAP:
                 if value := cfg.get('DEFAULT', CONFIG_ENV_MAP[key]):
-                    if key in ('ocis', 'screenRecordOnFailure'):
+                    if key in ('ocis', 'record_video_on_failure'):
                         CONFIG[key] = value == 'true'
                     else:
                         CONFIG[key] = value
@@ -128,7 +134,7 @@ def init_config():
     # read and override configs from environment variables
     for key, value in CONFIG_ENV_MAP.items():
         if os.environ.get(value):
-            if key in ('ocis', 'screenRecordOnFailure'):
+            if key in ('ocis', 'record_video_on_failure'):
                 CONFIG[key] = os.environ.get(value) == 'true'
             else:
                 CONFIG[key] = os.environ.get(value)
@@ -154,22 +160,19 @@ def init_config():
                 CONFIG[key] = value.rstrip('/') + '/'
 
 
-def get_config(key=None):
-    if key:
-        return CONFIG[key]
-    return CONFIG
+def get_config(key):
+    return CONFIG[key]
 
 
 def set_config(key, value):
     if key in READONLY_CONFIG:
         raise KeyError(f'Cannot set read-only config: {key}')
+    # save the initial config value
+    if key not in SCENARIO_CONFIGS:
+        SCENARIO_CONFIGS[key] = CONFIG.get(key)
     CONFIG[key] = value
 
 
 def clear_scenario_config():
-    global CONFIG
-    initial_config = {}
-    for key in READONLY_CONFIG:
-        initial_config[key] = CONFIG[key]
-
-    CONFIG = initial_config
+    for key, value in SCENARIO_CONFIGS.items():
+        CONFIG[key] = value

test/gui/shared/scripts/helpers/ReportHelper.py

@@ -0,0 +1,75 @@
+import os
+import glob
+import shutil
+import test
+import squish
+import squishinfo
+
+from helpers.ConfigHelper import get_config
+from helpers.FilesHelper import prefix_path_namespace
+
+
+def get_screenrecords_path():
+    return os.path.join(get_config("guiTestReportDir"), "screenrecords")
+
+
+def get_screenshots_path():
+    return os.path.join(get_config("guiTestReportDir"), "screenshots")
+
+
+def is_video_enabled():
+    return (
+        get_config("record_video_on_failure")
+        or get_config("retrying")
+        and not reached_video_limit()
+    )
+
+
+def reached_video_limit():
+    video_report_dir = get_screenrecords_path()
+    if not os.path.exists(video_report_dir):
+        return False
+    entries = [f for f in os.scandir(video_report_dir) if f.is_file()]
+    return len(entries) >= get_config("video_record_limit")
+
+
+def save_video_recording(filename, test_failed):
+    try:
+        # do not throw if stopVideoCapture() fails
+        test.stopVideoCapture()
+    except:
+        test.log("Failed to stop screen recording")
+
+    if not (video_dir := squishinfo.resultDir):
+        video_dir = squishinfo.testCase
+    else:
+        test_case = "/".join(squishinfo.testCase.split("/")[-2:])
+        video_dir = os.path.join(video_dir, test_case)
+    video_dir = os.path.join(video_dir, "attachments")
+
+    # if the test failed
+    # move videos to the screenrecords directory
+    if test_failed:
+        video_files = glob.glob(f"{video_dir}/**/*.mp4", recursive=True)
+        screenrecords_dir = get_screenrecords_path()
+        if not os.path.exists(screenrecords_dir):
+            os.makedirs(screenrecords_dir)
+        # reverse the list to get the latest video first
+        video_files.reverse()
+        for idx, video in enumerate(video_files):
+            if idx:
+                file_parts = filename.rsplit(".", 1)
+                filename = f"{file_parts[0]}_{idx+1}.{file_parts[1]}"
+            shutil.move(video, os.path.join(screenrecords_dir, filename))
+    # remove the video directory
+    shutil.rmtree(prefix_path_namespace(video_dir))
+
+
+def take_screenshot(filename):
+    directory = get_screenshots_path()
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+    try:
+        squish.saveDesktopScreenshot(os.path.join(directory, filename))
+    except:
+        test.log("Failed to save screenshot")

test/gui/shared/scripts/helpers/SetupClientHelper.py

@@ -13,6 +13,7 @@
 from helpers.SyncHelper import listen_sync_status_for_item
 from helpers.api.utils import url_join
 from helpers.UserHelper import get_displayname_for_user
+from helpers.ReportHelper import is_video_enabled
 
 
 def substitute_inline_codes(value):
@@ -103,8 +104,9 @@ def start_client():
         + ' --logdebug'
         + ' --logflush'
     )
-    if get_config('screenRecordOnFailure'):
+    if is_video_enabled():
         test.startVideoCapture()
+        set_config('video_recording_started', True)
 
 
 def get_polling_interval():