patch(integration_test_charm.yaml): Add Allure "unknown" status for t…

…ests with missing results (#183)

Generate default results with "unknown" status for every test so that if
Allure report is missing (because runner set up failed or integration
tests timed out) that those tests show up as "unknown" instead of being
omitted from Allure report

Breaking change for Allure users (Allure is in beta & not part of public
interface, hence `patch` PR prefix instead of `breaking`):
`allure-pytest-collection-report` plugin now required, see
.github/workflows/integration_test_charm_allure_beta.md

.github/workflows/integration_test_charm.yaml

@@ -150,11 +150,33 @@ jobs:
               print("Skipping unstable tests")
               output = "mark_expression=not unstable"
 
+          with open(os.environ["GITHUB_OUTPUT"], "a") as file:
+              file.write(output)
+      - name: (beta) Get Allure collection option
+        if: ${{ inputs._beta_allure_report }}
+        id: allure-collection-option
+        # TODO future improvement: check if allure-pytest installed instead
+        shell: python
+        run: |
+          import os
+
+          output = "option=--allure-collection-dir=allure-collection-default-results"
+
           with open(os.environ["GITHUB_OUTPUT"], "a") as file:
               file.write(output)
       - name: Collect test groups
         id: collect-groups
-        run: tox run -e integration -- tests/integration -m '${{ steps.select-test-stability.outputs.mark_expression }}' --collect-groups
+        run: tox run -e integration -- tests/integration -m '${{ steps.select-test-stability.outputs.mark_expression }}' --collect-groups ${{ steps.allure-collection-option.outputs.option }}
+      - name: (beta) Upload Allure collection results
+        # Default test results in case the integration tests time out or runner set up fails
+        # (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test)
+        if: ${{ inputs._beta_allure_report && github.event_name == 'schedule' && github.run_attempt == '1' }}
+        uses: actions/upload-artifact@v4
+        with:
+          # TODO future improvement: ensure artifact name is unique (if called with a matrix that changes inputs)
+          name: allure-collection-default-results-integration-test-charm
+          path: allure-collection-default-results/
+          if-no-files-found: error
     outputs:
       groups: ${{ steps.collect-groups.outputs.groups }}
       default_runner: ${{ steps.parse-architecture.outputs.default_runner }}
@@ -425,6 +447,15 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
+      - name: Get workflow version
+        id: workflow-version
+        uses: canonical/get-workflow-version-action@v1
+        with:
+          repository-name: canonical/data-platform-workflows
+          file-name: integration_test_charm.yaml
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install CLI
+        run: pipx install git+https://github.com/canonical/data-platform-workflows@'${{ steps.workflow-version.outputs.sha }}'#subdirectory=python/cli
       - name: Download Allure
         # Following instructions from https://allurereport.org/docs/gettingstarted-installation/#install-via-the-system-package-manager-for-linux
         run: gh release download --repo allure-framework/allure2 --pattern 'allure_*.deb'
@@ -448,12 +479,24 @@ jobs:
         with:
           ref: gh-pages-beta
           path: repo/
+      - name: Download default test collection results
+        # Default test results in case the integration tests time out or runner set up fails
+        # (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test)
+        uses: actions/download-artifact@v4
+        with:
+          path: allure-collection-default-results/
+          name: allure-collection-default-results-integration-test-charm
       - name: Download test results
         uses: actions/download-artifact@v4
         with:
           path: allure-results/
           pattern: allure-results-integration-test-charm-${{ inputs.cloud }}-juju-${{ inputs.juju-agent-version || needs.integration-test.outputs.juju-snap-channel-for-artifact }}-${{ inputs.architecture }}-*
           merge-multiple: true
+      - name: Combine Allure default results & actual results
+        # For every test: if actual result available, use that. Otherwise, use default result
+        # So that, if actual result not available, Allure report will show "unknown"/"failed" test result
+        # instead of omitting the test
+        run: allure-add-default-for-missing-results --allure-results-dir=allure-results --allure-collection-default-results-dir=allure-collection-default-results
       - name: Load test report history
         run: |
           if [[ -d repo/_latest/history/ ]]

.github/workflows/integration_test_charm_allure_beta.md

@@ -0,0 +1,27 @@
+[Allure Report](https://allurereport.org/) for [integration_test_charm.yaml](integration_test_charm.md)
+
+## Usage
+> [!WARNING]
+> This feature is in beta and **not part of the public interface**. It is subject to breaking changes or removal on a patch version bump.
+
+1. `poetry add --group integration allure-pytest`
+2. Add
+```toml
+allure-pytest-collection-report = {git = "https://github.com/canonical/data-platform-workflows", tag = "v0.0.0", subdirectory = "python/pytest_plugins/allure_pytest_collection_report"}
+```
+to your integration test dependencies in `pyproject.toml`.
+
+3. Set `_beta_allure_report: true` for **one** instance of `integration_test_charm.yaml`. If `integration_test_charm.yaml` is called with a matrix, `_beta_allure_report` can only be `true` for one combination of the matrix.
+4. Add permission to `integration_test_charm.yaml` job and all calling workflows
+```yaml
+    permissions:
+      contents: write  # Needed for Allure Report beta
+```
+5. Create gh pages branch
+https://github.com/canonical/data-platform-workflows/blob/5a2c81678ff8733345875235e579d0b1fffbc894/.github/workflows/integration_test_charm.yaml#L363-L371
+6. Enable gh pages publishing at https://github.com/canonical/mysql-router-k8s-operator/settings/pages (replace `mysql-router-k8s-operator` with repository name)
+    ![gh-pages](https://github.com/canonical/data-platform-workflows/assets/115640263/6ee80a1e-f75b-4d67-b11f-977358c32847)
+
+Example for 1, 3-4: https://github.com/canonical/mysql-router-k8s-operator/pull/198
+
+Example for 2:

python/cli/data_platform_workflows_cli/allure_add_default_for_missing_results.py

@@ -0,0 +1,49 @@
+import argparse
+import dataclasses
+import json
+import pathlib
+
+
+@dataclasses.dataclass(frozen=True)
+class Result:
+    test_case_id: str
+    path: pathlib.Path
+
+    def __eq__(self, other):
+        if not isinstance(other, type(self)):
+            return False
+        return self.test_case_id == other.test_case_id
+
+
+def main():
+    """Combine Allure default results & actual results
+
+    For every test: if actual result available, use that. Otherwise, use default result
+
+    So that, if actual result not available, Allure report will show "unknown"/"failed" test result
+    instead of omitting the test
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--allure-results-dir", required=True)
+    parser.add_argument("--allure-collection-default-results-dir", required=True)
+    args = parser.parse_args()
+
+    actual_results = pathlib.Path(args.allure_results_dir)
+    default_results = pathlib.Path(args.allure_collection_default_results_dir)
+
+    results: dict[pathlib.Path, set[Result]] = {
+        actual_results: set(),
+        default_results: set(),
+    }
+    for directory, results_ in results.items():
+        for path in directory.glob("*-result.json"):
+            with path.open("r") as file:
+                id_ = json.load(file)["testCaseId"]
+            results_.add(Result(id_, path))
+
+    actual_results.mkdir(exist_ok=True)
+
+    missing_results = results[default_results] - results[actual_results]
+    for default_result in missing_results:
+        # Move to `actual_results` directory
+        default_result.path.rename(actual_results / default_result.path.name)

python/cli/pyproject.toml

@@ -19,6 +19,7 @@ release-charm = "data_platform_workflows_cli.craft_tools.release:charm"
 update-bundle = "data_platform_workflows_cli.update_bundle:main"
 parse-snap-version = "data_platform_workflows_cli.parse_snap_version:main"
 convert-logsink-to-debug-log = "data_platform_workflows_cli.convert_logsink_to_debug_log:main"
+allure-add-default-for-missing-results = "data_platform_workflows_cli.allure_add_default_for_missing_results:main"
 
 [tool.poetry.dependencies]
 python = "^3.10"

python/pytest_plugins/allure_pytest_collection_report/.gitignore

@@ -0,0 +1 @@
+/poetry.lock

python/pytest_plugins/allure_pytest_collection_report/allure_pytest_collection_report/_plugin.py

@@ -0,0 +1,119 @@
+# Upstream feature request to replace this plugin:
+# https://github.com/allure-framework/allure-python/issues/821
+
+import allure_commons.logger
+import allure_commons.model2
+import allure_commons.types
+import allure_commons.utils
+import allure_pytest.listener
+import allure_pytest.utils
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--allure-collection-dir",
+        help="Generate default Allure results (used by GitHub Actions) in this directory for tests that are missing Allure results",
+    )
+
+
+def pytest_configure(config):
+    if config.option.allure_collection_dir:
+        config.option.collectonly = True
+
+
+def pytest_collection_finish(session):
+    report_dir = session.config.option.allure_collection_dir
+    if not report_dir:
+        return
+
+    # Copied from `allure_pytest.listener.AllureListener._cache`
+    _cache = allure_pytest.listener.ItemCache()
+    # Modified from `allure_pytest.plugin.pytest_configure`
+    file_logger = allure_commons.logger.AllureFileLogger(report_dir)
+
+    for item in session.items:
+        # Modified from `allure_pytest.listener.AllureListener.pytest_runtest_protocol`
+        uuid = _cache.push(item.nodeid)
+        test_result = allure_commons.model2.TestResult(name=item.name, uuid=uuid)
+
+        # Copied from `allure_pytest.listener.AllureListener.pytest_runtest_setup`
+        params = (
+            allure_pytest.listener.AllureListener._AllureListener__get_pytest_params(
+                item
+            )
+        )
+        test_result.name = allure_pytest.utils.allure_name(item, params)
+        full_name = allure_pytest.utils.allure_full_name(item)
+        test_result.fullName = full_name
+        test_result.testCaseId = allure_commons.utils.md5(full_name)
+        test_result.description = allure_pytest.utils.allure_description(item)
+        test_result.descriptionHtml = allure_pytest.utils.allure_description_html(item)
+        current_param_names = [param.name for param in test_result.parameters]
+        test_result.parameters.extend(
+            [
+                allure_commons.model2.Parameter(
+                    name=name, value=allure_commons.utils.represent(value)
+                )
+                for name, value in params.items()
+                if name not in current_param_names
+            ]
+        )
+
+        # Copied from `allure_pytest.listener.AllureListener.pytest_runtest_teardown`
+        test_result.historyId = allure_pytest.utils.get_history_id(
+            test_result.fullName,
+            test_result.parameters,
+            original_values=allure_pytest.listener.AllureListener._AllureListener__get_pytest_params(
+                item
+            ),
+        )
+        test_result.labels.extend(
+            [
+                allure_commons.model2.Label(name=name, value=value)
+                for name, value in allure_pytest.utils.allure_labels(item)
+            ]
+        )
+        test_result.labels.extend(
+            [
+                allure_commons.model2.Label(
+                    name=allure_commons.types.LabelType.TAG, value=value
+                )
+                for value in allure_pytest.utils.pytest_markers(item)
+            ]
+        )
+        allure_pytest.listener.AllureListener._AllureListener__apply_default_suites(
+            None, item, test_result
+        )
+        test_result.labels.append(
+            allure_commons.model2.Label(
+                name=allure_commons.types.LabelType.HOST,
+                value=allure_commons.utils.host_tag(),
+            )
+        )
+        test_result.labels.append(
+            allure_commons.model2.Label(
+                name=allure_commons.types.LabelType.FRAMEWORK, value="pytest"
+            )
+        )
+        test_result.labels.append(
+            allure_commons.model2.Label(
+                name=allure_commons.types.LabelType.LANGUAGE,
+                value=allure_commons.utils.platform_label(),
+            )
+        )
+        test_result.labels.append(
+            allure_commons.model2.Label(
+                name="package", value=allure_pytest.utils.allure_package(item)
+            )
+        )
+        test_result.links.extend(
+            [
+                allure_commons.model2.Link(link_type, url, name)
+                for link_type, url, name in allure_pytest.utils.allure_links(item)
+            ]
+        )
+
+        # Modified from `allure_pytest.listener.AllureListener.pytest_runtest_protocol`
+        test_result.status = allure_commons.model2.Status.UNKNOWN
+        # Modified from `allure_commons.reporter.AllureReporter.close_test`
+        file_logger.report_result(test_result)

python/pytest_plugins/allure_pytest_collection_report/pyproject.toml

@@ -0,0 +1,23 @@
+[tool.poetry]
+name = "allure-pytest-collection-report"
+# Version unused; repository has its own versioning system. (See .github/workflows/__release.yaml)
+version = "0.1.0"
+description = ""
+authors = ["Carl Csaposs <carl.csaposs@canonical.com>"]
+readme = "README.md"
+classifiers = [
+    "Framework :: Pytest",
+]
+
+[tool.poetry.plugins."pytest11"]
+allure_collection_report = "allure_pytest_collection_report._plugin"
+
+[tool.poetry.dependencies]
+python = "^3.8"
+pytest = "*"
+allure-pytest = ">=2.13.5"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"