Automated testing (#352)

TODO(kwk): Currently only dry-running tests.

.github/workflows/check-todays-snapshot.yml

@@ -63,6 +63,8 @@ jobs:
         shell: bash -e {0}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TESTING_FARM_API_TOKEN_PUBLIC_RANCH: ${{ secrets.TESTING_FARM_API_TOKEN_PUBLIC_RANCH }}
+          TESTING_FARM_API_TOKEN_REDHAT_RANCH: ${{ secrets.TESTING_FARM_API_TOKEN_REDHAT_RANCH }}
         run: |
           extra_args=""
 

snapshot_manager/snapshot_manager/build_status.py

@@ -6,12 +6,7 @@
 import enum
 import logging
 import pathlib
-import re
-import os
 
-import regex
-
-import snapshot_manager.build_status as build_status
 import snapshot_manager.util as util
 
 
@@ -141,55 +136,11 @@ def success(self) -> bool:
 
     @property
     def os(self):
-        """Get the os part of a chroot string
-
-        Args:
-            chroot (str): A string like "fedora-rawhide-x86_64
-
-        Returns:
-            str: The OS part of the chroot string.
-
-        Examples:
-
-        >>> BuildState(chroot="fedora-rawhide-x86_64").os
-        'fedora-rawhide'
-
-        >>> BuildState(chroot="fedora-40-ppc64le").os
-        'fedora-40'
-
-        >>> BuildState(chroot="fedora-rawhide-NEWARCH").os
-        'fedora-rawhide'
-        """
-        match = re.search(pattern=r"[^-]+-[0-9,rawhide]+", string=self.chroot)
-        if match:
-            return str(match[0])
-        return ""
+        return util.chroot_os(self.chroot)
 
     @property
     def arch(self):
-        """Get architecture part of a chroot string
-
-        Args:
-            chroot (str): A string like "fedora-rawhide-x86_64
-
-        Returns:
-            str: The architecture part of the chroot string.
-
-        Example:
-
-        >>> BuildState(chroot="fedora-rawhide-x86_64").arch
-        'x86_64'
-
-        >>> BuildState(chroot="fedora-40-ppc64le").arch
-        'ppc64le'
-
-        >>> BuildState(chroot="fedora-rawhide-NEWARCH").arch
-        'NEWARCH'
-        """
-        match = regex.search(pattern=r"[^-]+-[^-]+-\K[^\s]+", string=self.chroot)
-        if match:
-            return str(match[0])
-        return ""
+        return util.chroot_arch(self.chroot)
 
     @property
     def source_build_url(self) -> str:
@@ -390,8 +341,8 @@ def handle_golden_file(cause: ErrorCause, ctx: str):
     # TODO: Feel free to add your check here...
 
     logging.info(" Default to unknown cause...")
-    _, tail, _ = util._run_cmd(cmd=f"tail {build_log_file}")
-    _, rpm_build_errors, _ = util._run_cmd(
+    _, tail, _ = util.run_cmd(cmd=f"tail {build_log_file}")
+    _, rpm_build_errors, _ = util.run_cmd(
         cmd=rf"sed -n -e '/RPM build errors/,/Finish:/' p {build_log_file}"
     )
     _, errors_to_look_into, _ = util.grep_file(

snapshot_manager/snapshot_manager/config.py

@@ -56,6 +56,9 @@ class Config:
     )
     """URL to the Copr monitor page. We'll use this in the issue comment's body, not for querying Copr."""
 
+    test_repo_url: str = "https://github.com/fedora-llvm-team/llvm-snapshots"
+    """TBD"""
+
     @property
     def copr_projectname(self) -> str:
         """Takes the copr_project_tpl and replaces the YYYYMMDD placeholder (if any) with a date.
@@ -71,6 +74,11 @@ def copr_projectname(self) -> str:
         """
         return self.copr_project_tpl.replace("YYYYMMDD", self.yyyymmdd)
 
+    @property
+    def copr_project(self) -> str:
+        """Returns the owner/project string for the current date."""
+        return f"{self.config.copr_ownername}/{self.config.copr_projectname}"
+
     @property
     def copr_monitor_url(self) -> str:
         """Takes the copr_monitor_tpl and replaces the YYYYMMDD placeholder (if any) with a date.

snapshot_manager/snapshot_manager/copr_util.py

@@ -1,5 +1,5 @@
 """
-copr
+copr_util
 """
 
 import logging

snapshot_manager/snapshot_manager/github_util.py

@@ -1,5 +1,5 @@
 """
-github
+github_util
 """
 
 import datetime
@@ -176,3 +176,18 @@ def create_labels_for_strategies(self, labels: list[str], **kw_args):
 
     def create_labels_for_archs(self, labels: list[str], **kw_args):
         self._create_labels(labels=labels, prefix="arch/", color="C5DEF5", *kw_args)
+
+    def create_labels_for_in_testing(self, labels: list[str], **kw_args):
+        self._create_labels(
+            labels=labels, prefix="in_testing/", color="C2E0C6", *kw_args
+        )
+
+    def create_labels_for_tested_on(self, labels: list[str], **kw_args):
+        self._create_labels(
+            labels=labels, prefix="tested_on/", color="0E8A16", *kw_args
+        )
+
+    def create_labels_for_failed_on(self, labels: list[str], **kw_args):
+        self._create_labels(
+            labels=labels, prefix="failed_on/", color="D93F0B", *kw_args
+        )

snapshot_manager/snapshot_manager/snapshot_manager.py

@@ -4,11 +4,17 @@
 
 import datetime
 import logging
+import os
+import re
+
+import github.Issue
 
 import snapshot_manager.build_status as build_status
 import snapshot_manager.copr_util as copr_util
 import snapshot_manager.github_util as github_util
 import snapshot_manager.config as config
+import snapshot_manager.util as util
+import snapshot_manager.testing_farm_util as tf
 
 
 class SnapshotManager:
@@ -29,6 +35,8 @@ def check_todays_builds(self):
             )
             return
 
+        all_chroots = self.copr.get_copr_chroots()
+
         logging.info("Get build states from copr")
         states = self.copr.get_build_states_from_copr_monitor(
             copr_ownername=self.config.copr_ownername,
@@ -39,6 +47,11 @@ def check_todays_builds(self):
 
         comment_body = issue.body
 
+        logging.info(
+            "Extract and sanitize testing-farm information out of the last comment body."
+        )
+        testing_farm_requests = tf.parse_comment_for_request_ids(comment_body)
+
         logging.info(
             "Shorten the issue body comment to the update marker so that we can append to it"
         )
@@ -50,7 +63,7 @@ def check_todays_builds(self):
 
         logging.info("Add a build matrix")
         comment_body += build_status.markdown_build_status_matrix(
-            chroots=self.copr.get_copr_chroots(),
+            chroots=all_chroots,
             packages=self.config.packages,
             build_states=states,
         )
@@ -76,7 +89,7 @@ def check_todays_builds(self):
         arch_labels = list({f"arch/{err.arch}" for err in errors})
         strategy_labels = [f"strategy/{self.config.build_strategy}"]
         other_labels: list[str] = []
-        if errors is None or len(errors) > 0:
+        if errors is None and len(errors) > 0:
             other_labels.append("broken_snapshot_detected")
 
         logging.info("Create labels")
@@ -88,6 +101,9 @@ def check_todays_builds(self):
         self.github.create_labels_for_projects(project_labels)
         self.github.create_labels_for_archs(arch_labels)
         self.github.create_labels_for_strategies(strategy_labels)
+        self.github.create_labels_for_in_testing(all_chroots)
+        self.github.create_labels_for_tested_on(all_chroots)
+        self.github.create_labels_for_failed_on(all_chroots)
 
         # Remove old labels from issue if they no longer apply. This is greate
         # for restarted builds for example to make all builds green and be able
@@ -118,18 +134,149 @@ def check_todays_builds(self):
             logging.info(f"Adding label: {label}")
             issue.add_to_labels(label)
 
-        logging.info("Checking if issue can be closed")
-        all_good = self.copr.has_all_good_builds(
-            copr_ownername=self.config.copr_ownername,
-            copr_projectname=self.config.copr_projectname,
-            required_chroots=self.copr.get_copr_chroots(),
-            required_packages=self.config.packages,
-            states=states,
+        labels_on_issue = [label.name for label in issue.labels]
+
+        for chroot in all_chroots:
+            logging.info(f"Check if all builds in chroot {chroot} have succeeded")
+            builds_succeeded = self.copr.has_all_good_builds(
+                copr_ownername=self.config.copr_ownername,
+                copr_projectname=self.config.copr_projectname,
+                required_chroots=[chroot],
+                required_packages=self.config.packages,
+                states=states,
+            )
+
+            if not builds_succeeded:
+                continue
+
+            logging.info(f"All builds in chroot {chroot} have succeeded!")
+
+            if f"in_testing/{chroot}" in labels_on_issue:
+                logging.info(
+                    f"Chroot {chroot} is currently in testing! Not kicking off new tests."
+                )
+                if chroot in testing_farm_requests:
+                    request_id = testing_farm_requests[chroot]
+                    watch_result, artifacts_url = self.watch_testing_farm_request(
+                        request_id=request_id
+                    )
+                    if watch_result in [
+                        tf.TestingFarmWatchResult.TESTS_ERROR,
+                        tf.TestingFarmWatchResult.TESTS_FAILED,
+                    ]:
+                        issue.remove_from_labels(f"in_testing/{chroot}")
+                        issue.add_to_labels(f"failed_on/{chroot}")
+                    elif watch_result == tf.TestingFarmWatchResult.TESTS_PASSED:
+                        issue.remove_from_labels(f"in_testing/{chroot}")
+                        issue.add_to_labels(f"tested_on/{chroot}")
+
+            elif f"tested_on/{chroot}" in labels_on_issue:
+                logging.info(
+                    f"Chroot {chroot} has passed tests testing! Not kicking off new tests."
+                )
+            elif f"failed_on/{chroot}" in labels_on_issue:
+                logging.info(
+                    f"Chroot {chroot} has unsuccessful tests! Not kicking off new tests."
+                )
+            else:
+                logging.info(f"chroot {chroot} has no tests associated yet.")
+                request_id = self.make_testing_farm_request(chroot=chroot)
+                testing_farm_requests[chroot] = request_id
+                issue.add_to_labels(f"in_testing/{chroot}")
+
+        logging.info("Appending testing farm requests to the comment body.")
+        comment_body += "\n\n" + tf.chroot_request_ids_to_html_comment(
+            testing_farm_requests
         )
-        if all_good:
-            msg = f"@{self.config.maintainer_handle}, all required packages have been successfully built in all required chroots. We'll close this issue for you now as completed. Congratulations!"
+        issue.edit(body=comment_body)
+
+        logging.info("Checking if issue can be closed")
+        issue.update()
+        tested_chroot_labels = [
+            label.name for label in issue.labels if label.name.startswith("tested_on/")
+        ]
+        required_chroot_abels = ["tested_on/{chroot}" for chroot in all_chroots]
+        if set(tested_chroot_labels) == set(required_chroot_abels):
+            msg = f"@{self.config.maintainer_handle}, all required packages have been successfully built and tested on all required chroots. We'll close this issue for you now as completed. Congratulations!"
             logging.info(msg)
             issue.create_comment(body=msg)
             issue.edit(state="closed", state_reason="completed")
+            # TODO(kwk): Promotion of issue goes here.
+        else:
+            logging.info("Cannot close issue yet.")
 
         logging.info(f"Updated today's issue: {issue.html_url}")
+
+    def watch_testing_farm_request(
+        self, request_id: str
+    ) -> tuple[tf.TestingFarmWatchResult, str]:
+        request_id = tf.sanitize_request_id(request_id=request_id)
+        cmd = f"testing-farm watch --no-wait --id {request_id}"
+        exit_code, stdout, stderr = util.run_cmd(cmd=cmd)
+        if exit_code != 0:
+            raise SystemError(
+                f"failed to watch 'testing-farm request': {cmd}\n\nstdout: {stdout}\n\nstderr: {stderr}"
+            )
+
+        watch_result, artifacts_url = tf.parse_for_watch_result(stdout)
+        if watch_result is None:
+            raise SystemError(
+                f"failed to watch 'testing-farm request': {cmd}\n\nstdout: {stdout}\n\nstderr: {stderr}"
+            )
+        return (watch_result, artifacts_url)
+
+    def make_testing_farm_request(self, chroot: str) -> str:
+        """Runs a "testing-farm request" command and returns the request ID.
+
+        The request is made without waiting for the result.
+        It is the responsibility of the caller of this function to run "testing-farm watch --id <REQUEST_ID>",
+        where "<REQUEST_ID>" is the result of this function.
+
+        Depending on the chroot, we'll automatically select the proper testing-farm ranch for you.
+        For this to work you'll have to set the
+        TESTING_FARM_API_TOKEN_PUBLIC_RANCH and
+        TESTING_FARM_API_TOKEN_REDHAT_RANCH
+        environment variables. We'll then use one of them to set the TESTING_FARM_API_TOKEN
+        environment variable for the actual call to testing-farm.
+
+        Args:
+            chroot (str): The chroot that you want to run tests for.
+
+        Raises:
+            SystemError: When the testing-farm request failed
+
+        Returns:
+            str: Request ID
+        """
+        logging.info(f"Kicking off new tests for chroot {chroot}.")
+        all_tests_succeeded = False
+
+        # TODO(kwk): Add testing-farm code here, something like this:
+        # TODO(kwk): Decide how if we want to wait for test results (probably not) and if not how we can check for the results later.
+        ranch = tf.select_ranch(chroot)
+        logging.info(f"Using testing-farm ranch: {ranch}")
+        if ranch == "public":
+            os.environ["TESTING_FARM_API_TOKEN"] = os.getenv(
+                "TESTING_FARM_API_TOKEN_PUBLIC_RANCH"
+            )
+        if ranch == "redhat":
+            os.environ["TESTING_FARM_API_TOKEN"] = os.getenv(
+                "TESTING_FARM_API_TOKEN_REDHAT_RANCH"
+            )
+        cmd = f"""testing-farm \
+            request \
+            --compose {util.chroot_os(chroot).capitalize()} \
+            --git-url {self.config.test_repo_url} \
+            --arch {util.chroot_arch(chroot)} \
+            --plan /tests/snapshot-gating \
+            --environment COPR_PROJECT={self.config.copr_projectname} \
+            --context distro={util.chroot_os(chroot)} \
+            --context arch=${util.chroot_arch(chroot)} \
+            --no-wait \
+            --context snapshot={self.config.yyyymmdd}"""
+        exit_code, stdout, stderr = util.run_cmd(cmd, timeout_secs=None)
+        if exit_code == 0:
+            return tf.parse_output_for_request_id(stdout)
+        raise SystemError(
+            f"failed to run 'testing-farm request': {cmd}\n\nstdout: {stdout}\n\nstderr: {stderr}"
+        )