From e15c4e84f0a250265440940f5a230a0e98d87304 Mon Sep 17 00:00:00 2001
From: matejcik <ja@matejcik.cz>
Date: Sat, 30 Mar 2024 13:55:00 +0100
Subject: [PATCH] feat(tests): reduce ui error spam

---
 tests/conftest.py          | 17 ++++++++++++++++-
 tests/ui_tests/__init__.py | 37 ++++++++++++++++---------------------
 tests/ui_tests/common.py   |  5 +++--
 3 files changed, 35 insertions(+), 24 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 5d2147118b6..74c1cbb736c 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -18,10 +18,11 @@
 
 import os
 from pathlib import Path
-from typing import TYPE_CHECKING, Generator, Iterator
+from typing import TYPE_CHECKING, Callable, Generator, Iterator
 
 import pytest
 import xdist
+from _pytest.reports import TestReport
 
 from trezorlib import debuglink, log, models
 from trezorlib.debuglink import TrezorClientDebugLink as Client
@@ -451,6 +452,20 @@ def pytest_runtest_makereport(item: pytest.Item, call) -> Generator:
     setattr(item, f"rep_{rep.when}", rep)
 
 
+@pytest.hookimpl(tryfirst=True)
+def pytest_report_teststatus(
+    report: TestReport, config: Config
+) -> tuple[str, str, tuple[str, dict[str, bool]]] | None:
+    if report.passed:
+        for prop, _ in report.user_properties:
+            if prop == "ui_failed":
+                return "ui_failed", "U", ("UI-FAILED", {"red": True})
+            if prop == "ui_missing":
+                return "ui_missing", "M", ("UI-MISSING", {"yellow": True})
+    # else use default handling
+    return None
+
+
 @pytest.fixture
 def device_handler(client: Client, request: pytest.FixtureRequest) -> Generator:
     device_handler = BackgroundDeviceHandler(client)
diff --git a/tests/ui_tests/__init__.py b/tests/ui_tests/__init__.py
index f4a6b8ff802..a6fb0134918 100644
--- a/tests/ui_tests/__init__.py
+++ b/tests/ui_tests/__init__.py
@@ -5,6 +5,7 @@
 from typing import Callable, Generator
 
 import pytest
+from _pytest.nodes import Node
 from _pytest.outcomes import Failed
 
 from trezorlib.debuglink import TrezorClientDebugLink as Client
@@ -23,23 +24,13 @@ def _process_recorded(result: TestResult) -> None:
     testreport.recorded(result)
 
 
-def _process_tested(result: TestResult) -> None:
+def _process_tested(result: TestResult, item: Node) -> None:
     if result.expected_hash is None:
-        file_path = testreport.missing(result)
-        pytest.fail(
-            f"Hash of {result.test.id} not found in fixtures.json\n"
-            f"Expected:  {result.expected_hash}\n"
-            f"Actual:    {result.actual_hash}\n"
-            f"Diff file: {file_path}"
-        )
+        testreport.missing(result)
+        item.user_properties.append(("ui_missing", None))
     elif result.actual_hash != result.expected_hash:
-        file_path = testreport.failed(result)
-        pytest.fail(
-            f"Hash of {result.test.id} differs\n"
-            f"Expected:  {result.expected_hash}\n"
-            f"Actual:    {result.actual_hash}\n"
-            f"Diff file: {file_path}"
-        )
+        testreport.failed(result)
+        item.user_properties.append(("ui_failed", None))
     else:
         testreport.passed(result)
 
@@ -83,7 +74,7 @@ def screen_recording(
     if test_ui == "record":
         _process_recorded(result)
     else:
-        _process_tested(result)
+        _process_tested(result, request.node)
 
 
 def setup(main_runner: bool) -> None:
@@ -156,6 +147,9 @@ def terminal_summary(
 
     if normal_exit:
         println("-------- UI tests summary: --------")
+        for result in TestResult.recent_results():
+            if result.passed and not result.ui_passed:
+                println(f"UI_FAILED: {result.test.id} ({result.actual_hash})")
         println("Run ./tests/show_results.py to open test summary")
         println("")
 
@@ -176,15 +170,16 @@ def sessionfinish(
 
     testreport.generate_reports(record_text_layout, do_master_diff)
 
+    recents = list(TestResult.recent_results())
+
     if test_ui == "test":
-        common.write_fixtures_only_new_results(
-            TestResult.recent_results(),
-            dest=FIXTURES_RESULTS_FILE,
-        )
+        common.write_fixtures_only_new_results(recents, dest=FIXTURES_RESULTS_FILE)
+        if any(t.passed and not t.ui_passed for t in recents):
+            return pytest.ExitCode.TESTS_FAILED
 
     if test_ui == "test" and check_missing and list_missing():
         common.write_fixtures_complete(
-            TestResult.recent_results(),
+            recents,
             remove_missing=True,
             dest=FIXTURES_SUGGESTION_FILE,
         )
diff --git a/tests/ui_tests/common.py b/tests/ui_tests/common.py
index 9b9b7834a34..4989ab1490a 100644
--- a/tests/ui_tests/common.py
+++ b/tests/ui_tests/common.py
@@ -322,7 +322,8 @@ def save_metadata(self) -> None:
             json.dumps(metadata, indent=2, sort_keys=True) + "\n"
         )
 
-    def succeeded_in_ui_comparison(self) -> bool:
+    @property
+    def ui_passed(self) -> bool:
         return self.actual_hash == self.expected_hash
 
     @classmethod
@@ -354,7 +355,7 @@ def recent_results(cls) -> t.Iterator[Self]:
     def recent_ui_failures(cls) -> t.Iterator[Self]:
         """Returning just the results that resulted in UI failure."""
         for result in cls.recent_results():
-            if not result.succeeded_in_ui_comparison():
+            if not result.ui_passed:
                 yield result
 
     def store_recorded(self) -> None: