ci: actions: translations label trigger, update_fixtures.py support

.github/actions/ui-report/action.yml

@@ -4,6 +4,9 @@ inputs:
   model:
     description: 'Internal model name'
     required: true
+  lang:
+    description: 'Firmware language code'
+    required: true
   status:
     description: 'Either "success" or "failure"'
     required: false
@@ -17,7 +20,7 @@ runs:
         role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data
         aws-region: eu-west-1
     - run: |
-        MODELJOB=${{ inputs.model }}-${{ github.job }}
+        MODELJOB=${{ inputs.model }}-${{ inputs.lang }}-${{ github.job }}
         OUTDIR=${{ github.run_id }}/$MODELJOB
         mkdir -p $OUTDIR
         nix-shell --run "poetry run python ci/prepare_ui_artifacts.py || true"
@@ -36,6 +39,7 @@ runs:
     - name: Upload report
       run: |
         aws s3 sync ${{ github.run_id }} s3://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}
+        echo "[UI test report](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/${{ inputs.model }}-${{ inputs.lang }}-${{ github.job }}/index.html)" >> $GITHUB_STEP_SUMMARY
       shell: sh
     - name: Upload test screen recording
       run: |
@@ -46,6 +50,6 @@ runs:
       with:
         name: ui-records-${{ github.job }}-${{ strategy.job-index }}
         path: |
-          # used by core_ui_master
-          screens_${{ inputs.model }}-${{ github.job }}.tar
+          # used by core_ui_main
+          screens_${{ inputs.model }}-${{ inputs.lang }}-${{ github.job }}.tar
         retention-days: 1  # not useful after workflow finishes

.github/workflows/core.yml

@@ -20,9 +20,9 @@ env:
   PULL_COMMENT: |
     |core UI changes|device test|click test|persistence test|
     |---------------|-----------|----------|----------------|
-    |T2T1 Model T   |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_device_test/status.png" width="24" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_device_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_device_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_device_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_device_test/master_diff.html)) |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_click_test/status.png" width="20px" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_click_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_click_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_click_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_click_test/master_diff.html)) |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_persistence_test/status.png" width="20px" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_persistence_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_persistence_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_persistence_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-core_persistence_test/master_diff.html))||
-    |T2B1 Safe 3    |[3280](https://github.com/trezor/trezor-firmware/issues/3280)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_click_test/status.png" width="20px" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_click_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_click_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_click_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-core_click_test/master_diff.html)) |[2724](https://github.com/trezor/trezor-firmware/issues/2724)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    ||
-    |T3T1           |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_device_test/status.png" width="20px" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_device_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_device_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_device_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_device_test/master_diff.html)) |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_click_test/status.png" width="20px" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_click_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_click_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_click_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_click_test/master_diff.html)) |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_persistence_test/status.png" width="20px" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_persistence_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_persistence_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_persistence_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-core_persistence_test/master_diff.html))||
+    |T2T1 Model T   |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_device_test/status.png" width="24" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_device_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_device_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_device_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_device_test/master_diff.html)) |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test/status.png" width="20px" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_click_test/master_diff.html)) |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test/status.png" width="20px" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2T1-en-core_persistence_test/master_diff.html))||
+    |T2B1 Safe 3    |[3280](https://github.com/trezor/trezor-firmware/issues/3280)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-en-core_click_test/status.png" width="20px" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-en-core_click_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-en-core_click_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-en-core_click_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T2B1-en-core_click_test/master_diff.html)) |[2724](https://github.com/trezor/trezor-firmware/issues/2724)                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    ||
+    |T3T1           |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_device_test/status.png" width="20px" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_device_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_device_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_device_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_device_test/master_diff.html)) |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test/status.png" width="20px" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_click_test/master_diff.html)) |<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test/status.png" width="20px" height="20px" /> [test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test/differing_screens.html)) [main](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test/master_index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T3T1-en-core_persistence_test/master_diff.html))||
     |All            |<img src="https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/status.png" width="20px" height="20px" /> [main](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/index.html)([screens](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/master_diff.html)) ||
 
 jobs:
@@ -36,7 +36,7 @@ jobs:
       - id: set_vars
         name: Set variables
         run: |
-          echo test_lang=${{ github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'translations' && '[\"en\", \"cs\", \"fr\", \"de\", \"es\"]' || '[\"en\"]' }} >> $GITHUB_OUTPUT
+          echo test_lang=${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'translations') && '[\"en\", \"cs\", \"fr\", \"de\", \"es\"]' || '[\"en\"]' }} >> $GITHUB_OUTPUT
           echo asan=${{ github.event_name == 'schedule' && '[\"noasan\", \"asan\"]' || '[\"noasan\"]' }} >> $GITHUB_OUTPUT
           cat $GITHUB_OUTPUT
 
@@ -251,6 +251,7 @@ jobs:
       - uses: ./.github/actions/ui-report
         with:
           model: ${{ matrix.model }}
+          lang: ${{ matrix.lang }}
           status: ${{ job.status }}
         if: ${{ always() && env.ACTIONS_DO_UI_TEST == 'true' }}
         continue-on-error: true
@@ -305,6 +306,7 @@ jobs:
       - uses: ./.github/actions/ui-report
         with:
           model: ${{ matrix.model }}
+          lang: ${{ matrix.lang }}
           status: ${{ job.status }}
         if: always()
         continue-on-error: true
@@ -379,6 +381,7 @@ jobs:
       - uses: ./.github/actions/ui-report
         with:
           model: ${{ matrix.model }}
+          lang: en
           status: ${{ job.status }}
         if: always()
         continue-on-error: true

.github/workflows/legacy.yml

@@ -11,7 +11,7 @@ permissions:
 
 env:
   PULL_COMMENT: |
-    |legacy UI changes|<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-legacy_device_test/status.png" width="20px" height="20px" /> [device test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-legacy_device_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-legacy_device_test/differing_screens.html)) <img src="https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/status.png" width="20px" height="20px" /> [main](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/)([screens](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/master_diff.html))|
+    |legacy UI changes|<img src="https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-en-legacy_device_test/status.png" width="20px" height="20px" /> [device test](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-en-legacy_device_test/index.html)([screens](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/T1B1-en-legacy_device_test/differing_screens.html)) <img src="https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/status.png" width="20px" height="20px" /> [main](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/)([screens](https://data.trezor.io/dev/firmware/master_diff/${{ github.run_id }}/master_diff.html))|
     |-----------------|-----|
 
 jobs:
@@ -97,6 +97,7 @@ jobs:
       - uses: ./.github/actions/ui-report
         with:
           model: T1B1
+          lang: en
           status: ${{ job.status }}
         continue-on-error: true
         if: ${{ always() && matrix.coins == 'universal' }}

ci/make_nonenglish_tests.py

@@ -145,17 +145,17 @@ def get_all_tests_text(func: Callable[[str, str], str]) -> str:
 
 def fill_device_tests() -> None:
     replacement = Replacement(
-        start=r"##START_DEVICE_TESTS",
-        end=r"##END_DEVICE_TESTS",
+        start=r"## START_DEVICE_TESTS",
+        end=r"## END_DEVICE_TESTS",
         replacement=get_all_tests_text(get_device_test),
     )
     replace_content_between_markers(TEST_FILE, [replacement])
 
 
 def fill_click_tests() -> None:
     replacement = Replacement(
-        start=r"##START_CLICK_TESTS",
-        end=r"##END_CLICK_TESTS",
+        start=r"## START_CLICK_TESTS",
+        end=r"## END_CLICK_TESTS",
         replacement=get_all_tests_text(get_click_test),
     )
     replace_content_between_markers(TEST_FILE, [replacement])

ci/test-nonenglish.yml

@@ -10,7 +10,7 @@ image: registry.gitlab.com/satoshilabs/trezor/trezor-firmware/trezor-firmware-en
     paths:
       - .venv/
 
-##START_DEVICE_TESTS
+## START_DEVICE_TESTS
 core device test czech:
   stage: test
   <<: *gitlab_caching
@@ -300,9 +300,9 @@ core device R test spanish:
       junit: tests/junit.xml
 
 
-##END_DEVICE_TESTS
+## END_DEVICE_TESTS
 
-##START_CLICK_TESTS
+## START_CLICK_TESTS
 core click test czech:
   stage: test
   <<: *gitlab_caching
@@ -584,4 +584,4 @@ core click R test spanish:
     when: always
 
 
-##END_CLICK_TESTS
+## END_CLICK_TESTS

tests/github.py

@@ -0,0 +1,66 @@
+"""
+Helper functions for communication with GitHub.
+
+Allowing for interaction with the test results, e.g. with UI tests.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Iterable, Iterator
+
+import requests
+
+AnyDict = dict[Any, Any]
+
+HERE = Path(__file__).parent
+
+LIST_RUNS_TEMPLATE = "https://api.github.com/repos/trezor/trezor-firmware/actions/workflows/{workflow}/runs?branch={branch}"
+FIXTURES_TEMPLATE = "https://data.trezor.io/dev/firmware/ui_report/{run}/{model}-{lang}-{job}/fixtures.results.json"
+
+MODELS = ["T2T1", "T2B1", "T3T1"]
+LANGUAGES = ["en", "cs", "de", "es", "fr"]
+JOBS = ["core_device_test", "core_click_test", "core_persistence_test"]
+
+
+def get_branch_ui_fixtures_results(
+    branch_name: str,
+    only_jobs: Iterable[str] | None,
+    exclude_jobs: Iterable[str] | None,
+) -> dict[str, AnyDict]:
+    print(f"Checking branch {branch_name}")
+
+    response = requests.get(
+        LIST_RUNS_TEMPLATE.format(branch=branch_name, workflow="core.yml")
+    )
+    response.raise_for_status()
+    run_id = response.json()["workflow_runs"][0]["id"]
+
+    def yield_key_value() -> Iterator[tuple[str, AnyDict]]:
+        for model in MODELS:
+            for lang in LANGUAGES:
+                for job in JOBS:
+                    job_instance = f"{model}-{lang}-{job}"
+
+                    if only_jobs and all(
+                        (job not in job_instance) for job in only_jobs
+                    ):
+                        continue
+                    if exclude_jobs and any(
+                        (job in job_instance) for job in exclude_jobs
+                    ):
+                        continue
+
+                    response = requests.get(
+                        FIXTURES_TEMPLATE.format(
+                            run=run_id, model=model, lang=lang, job=job
+                        )
+                    )
+                    if response.status_code != 200:
+                        print(
+                            f"Failed to get fixtures for {job_instance}: {response.status_code}"
+                        )
+                        continue
+                    yield job_instance, response.json()
+
+    return dict(yield_key_value())