Merge branch 'main' of https://github.com/EESSI/eessi-bot-software-layer

 into update_PR182

README.md

@@ -354,6 +354,27 @@ submit_command = /usr/bin/sbatch
 ```
 This is the full path to the Slurm command used for submitting batch jobs. You may want to verify if `sbatch` is provided at that path or determine its actual location (`which sbatch`).
 
+### Section `[bot_control]`
+The section `[bot_control]` contains settings for configuring the feature to
+send commands to the bot.
+```
+command_permission = GH_ACCOUNT_1 GH_ACCOUNT_2 ...
+```
+The option `command_permission` defines which GitHub accounts can send commands
+to the bot (via new PR comments). If the value is empty NO account can send
+commands.
+
+```
+command_response_fmt = FORMAT_MARKDOWN_AND_HTML
+```
+This allows to customize the format of the comments about the handling of bot
+commands. The format needs to include `{sender}`, `{comment_response}` and
+`{comment_result}`. `{app_name}` is replaced with the name of the bot instance.
+`{comment_response}` is replaced with information about parsing the comment
+for commands before any command is run. `{comment_result}` is replaced with
+information about the result of the command that was run (can be empty).
+
+
 ### Section `[deploycfg]`
 The section `[deploycfg]` defines settings for uploading built artefacts (tarballs).
 ```

app.cfg.example

@@ -24,6 +24,26 @@ installation_id = 12345678
 private_key = PATH_TO_PRIVATE_KEY
 
 
+[bot_control]
+# which GH accounts have the permission to send commands to the bot
+# if value is left/empty everyone can send commands
+# value can be a space delimited list of GH accounts
+# NOTE, be careful to not list the bot's name (GH app name) or it may consider
+#   comments created/updated by itself as a bot command and thus enter an
+#   endless loop.
+command_permission =
+
+# format of the response when processing bot commands. NOTE, make sure the
+# placeholders {app_name}, {comment_response} and {comment_result} are included.
+command_response_fmt =
+    <details><summary>Updates by the bot instance <code>{app_name}</code>
+    <em>(click for details)</em></summary>
+
+    {comment_response}
+    {comment_result}
+    </details>
+
+
 [buildenv]
 # name of the job script used for building an EESSI stack
 build_job_script = PATH_TO_EESSI_BOT/scripts/bot-build.slurm

eessi_bot_event_handler.py

@@ -19,15 +19,26 @@
 import tasks.deploy as deploy
 
 from connections import github
+from tasks.build import check_build_permission, get_architecture_targets, get_repo_cfg, submit_build_jobs
+from tasks.deploy import deploy_built_artefacts
 from tools import config
 from tools.args import event_handler_parse
-from tasks.build import check_build_permission, submit_build_jobs, get_repo_cfg
-from tasks.deploy import deploy_built_artefacts
+from tools.commands import EESSIBotCommand, EESSIBotCommandError, get_bot_command
+from tools.filter import EESSIBotActionFilter
+from tools.permissions import check_command_permission
+from tools.pr_comments import create_comment
 
 from pyghee.lib import PyGHee, create_app, get_event_info, read_event_from_json
 from pyghee.utils import log
 
 
+APP_NAME = "app_name"
+BOT_CONTROL = "bot_control"
+COMMAND_RESPONSE_FMT = "command_response_fmt"
+GITHUB = "github"
+REPO_TARGET_MAP = "repo_target_map"
+
+
 class EESSIBotSoftwareLayer(PyGHee):
 
     def __init__(self, *args, **kwargs):
@@ -60,10 +71,151 @@ def handle_issue_comment_event(self, event_info, log_file=None):
         """
         request_body = event_info['raw_request_body']
         issue_url = request_body['issue']['url']
-        comment_author = request_body['comment']['user']['login']
-        comment_txt = request_body['comment']['body']
-        self.log("Comment posted in %s by @%s: %s", issue_url, comment_author, comment_txt)
-        self.log("issue_comment event handled!")
+        action = request_body['action']
+        sender = request_body['sender']['login']
+        owner = request_body['comment']['user']['login']
+        repo_name = request_body['repository']['full_name']
+        pr_number = request_body['issue']['number']
+
+        # FIXME add request body text (['comment']['body']) to log message when
+        # log level is set to debug
+        self.log(f"Comment in {issue_url} (owned by @{owner}) {action} by @{sender}")
+
+        # obtain app name and format for reporting about received & processed
+        # commands
+        app_name = self.cfg[GITHUB][APP_NAME]
+        command_response_fmt = self.cfg[BOT_CONTROL][COMMAND_RESPONSE_FMT]
+
+        # currently, only commands in new comments are supported
+        #  - commands have the syntax 'bot: COMMAND [ARGS*]'
+
+        # first check if sender is authorized to send any command
+        # - this serves a double purpose:
+        #   1. check permission
+        #   2. skip any comment updates that were done by the bot itself
+        #      --> thus we prevent the bot from entering an endless loop
+        #          where it reacts on updates to comments it made itself
+        #      NOTE this assumes that the sender of the event is corresponding to
+        #      the bot if the bot updates comments itself and that the bot is not
+        #      given permission in the configuration setting 'command_permission'
+        #      ... in order to prevent surprises we should be careful what the bot
+        #      adds to comments, for example, before updating a comment it could
+        #      run the update through the function checking for a bot command.
+        if check_command_permission(sender) is False:
+            self.log(f"account `{sender}` has NO permission to send commands to bot")
+            # need to ensure that the bot is not responding on its own comments
+            # as a quick implementation we check if the sender name contains '[bot]'
+            # FIXME improve this by querying (and caching) information about the sender of
+            #       an event ALTERNATIVELY we could postpone this test a bit until we
+            #       have parsed the comment and know if it contains any bot command
+            if not sender.endswith('[bot]'):
+                comment_response = f"\n- account `{sender}` has NO permission to send commands to the bot"
+                comment_body = command_response_fmt.format(
+                    app_name=app_name,
+                    comment_response=comment_response,
+                    comment_result=''
+                )
+                issue_comment = create_comment(repo_name, pr_number, comment_body)
+            else:
+                self.log(f"account `{sender}` seems to be a bot instance itself, hence not creating a new PR comment")
+            return
+        else:
+            self.log(f"account `{sender}` has permission to send commands to bot")
+
+        # only scan for commands in newly created comments
+        if action == 'created':
+            comment_received = request_body['comment']['body']
+            self.log(f"comment action '{action}' is handled")
+        else:
+            # NOTE we do not report this with a new PR comment, because otherwise any update to a comment would
+            # let the bot add a response (would be very noisy); we might add a debug mode later
+            self.log(f"comment action '{action}' not handled")
+            return
+
+        # search for commands in comment
+        comment_response = ''
+        commands = []
+        # process non-empty (if x) lines (split) in comment
+        for line in [x for x in [y.strip() for y in comment_received.split('\n')] if x]:
+            # FIXME add processed line(s) to log when log level is set to debug
+            bot_command = get_bot_command(line)
+            if bot_command:
+                try:
+                    ebc = EESSIBotCommand(bot_command)
+                except EESSIBotCommandError as bce:
+                    self.log(f"ERROR: parsing bot command '{bot_command}' failed with {bce.args}")
+                    # FIXME possibly add more information to log when log level is set to debug
+                    comment_response += f"\n- parsing the bot command `{bot_command}`, received"
+                    comment_response += f" from sender `{sender}`, failed"
+                    continue
+                commands.append(ebc)
+                self.log(f"found bot command: '{bot_command}'")
+                comment_response += f"\n- received bot command `{bot_command}`"
+                comment_response += f" from `{sender}`"
+                comment_response += f"\n  - expanded format: `{ebc.to_string()}`"
+            # FIXME add an else branch that logs information for comments not
+            # including a bot command; the logging should only be done when log
+            # level is set to debug
+
+        if comment_response == '':
+            # no update to be added, just log and return
+            self.log("comment response is empty")
+            return
+        else:
+            self.log(f"comment response: '{comment_response}'")
+
+        if not any(map(get_bot_command, comment_response.split('\n'))):
+            # the 'not any()' ensures that the response would not be considered a bot command itself
+            # ... together with checking the sender of a comment update this aims
+            # at preventing the bot to enter an endless loop in commenting on its own
+            # comments
+            comment_body = command_response_fmt.format(
+                app_name=app_name,
+                comment_response=comment_response,
+                comment_result=''
+            )
+            issue_comment = create_comment(repo_name, pr_number, comment_body)
+        else:
+            self.log(f"update '{comment_response}' is considered to contain bot command ... not creating PR comment")
+            # FIXME we may want to report this back to the PR on GitHub, e.g.,
+            # "Oops response message seems to contain a bot command. It is not
+            # displayed here to prevent the bot from entering an endless loop
+            # of commands. Please, check the logs at the bot instance for more
+            # information."
+
+        # process commands
+        comment_result = ''
+        for cmd in commands:
+            try:
+                update = self.handle_bot_command(event_info, cmd)
+                comment_result += f"\n- handling command `{cmd.to_string()}` resulted in: "
+                comment_result += update
+                self.log(f"handling command '{cmd.to_string()}' resulted in '{update}'")
+
+            except EESSIBotCommandError as err:
+                self.log(f"ERROR: handling command {cmd.command} failed with {err.args[0]}")
+                comment_result += f"\n- handling command `{cmd.command}` failed with message"
+                comment_result += f"\n  _{err.args[0]}_"
+                continue
+            except Exception as err:
+                log(f"Unexpected err={err}, type(err)={type(err)}")
+                if comment_result:
+                    comment_body = command_response_fmt.format(
+                        app_name=app_name,
+                        comment_response=comment_response,
+                        comment_result=comment_result
+                    )
+                    issue_comment.edit(comment_body)
+                raise
+        # only update PR comment once
+        comment_body = command_response_fmt.format(
+            app_name=app_name,
+            comment_response=comment_response,
+            comment_result=comment_result
+        )
+        issue_comment.edit(comment_body)
+
+        self.log(f"issue_comment event (url {issue_url}) handled!")
 
     def handle_installation_event(self, event_info, log_file=None):
         """
@@ -88,7 +240,8 @@ def handle_pull_request_labeled_event(self, event_info, pr):
         if label == "bot:build":
             # run function to build software stack
             if check_build_permission(pr, event_info):
-                submit_build_jobs(pr, event_info)
+                # use an empty filter
+                submit_build_jobs(pr, event_info, EESSIBotActionFilter(""))
 
         elif label == "bot:deploy":
             # run function to deploy built artefacts
@@ -101,13 +254,20 @@ def handle_pull_request_opened_event(self, event_info, pr):
         Handle opening of a pull request.
         """
         self.log("PR opened: waiting for label bot:build")
-        app_name = self.cfg['github']['app_name']
+        app_name = self.cfg[GITHUB][APP_NAME]
         # TODO check if PR already has a comment with arch targets and
         # repositories
+        arch_map = get_architecture_targets(self.cfg)
         repo_cfg = get_repo_cfg(self.cfg)
+
         comment = f"Instance `{app_name}` is configured to build:"
-        for arch in repo_cfg['repo_target_map'].keys():
-            for repo_id in repo_cfg['repo_target_map'][arch]:
+
+        for arch in arch_map.keys():
+            # check if repo_target_map contains an entry for {arch}
+            if arch not in repo_cfg[REPO_TARGET_MAP]:
+                self.log(f"skipping arch {arch} because repo target map does not define repositories to build for")
+                continue
+            for repo_id in repo_cfg[REPO_TARGET_MAP][arch]:
                 comment += f"\n- arch `{'/'.join(arch.split('/')[1:])}` for repo `{repo_id}`"
 
         self.log(f"PR opened: comment '{comment}'")
@@ -117,7 +277,8 @@ def handle_pull_request_opened_event(self, event_info, pr):
         gh = github.get_instance()
         repo = gh.get_repo(repo_name)
         pull_request = repo.get_pull(pr.number)
-        pull_request.create_issue_comment(comment)
+        issue_comment = pull_request.create_issue_comment(comment)
+        return issue_comment
 
     def handle_pull_request_event(self, event_info, log_file=None):
         """
@@ -138,6 +299,67 @@ def handle_pull_request_event(self, event_info, log_file=None):
         else:
             self.log("No handler for PR action '%s'", action)
 
+    def handle_bot_command(self, event_info, bot_command, log_file=None):
+        """
+        Handle bot command
+
+        Args:
+            event_info (dict): object containing all information of the event
+            bot_command (EESSIBotCommand): command to be handled
+        """
+        cmd = bot_command.command
+        handler_name = f"handle_bot_command_{cmd}"
+        if hasattr(self, handler_name):
+            handler = getattr(self, handler_name)
+            self.log(f"Handling bot command {cmd}")
+            return handler(event_info, bot_command)
+        else:
+            self.log(f"No handler for command '{cmd}'")
+            raise EESSIBotCommandError(f"unknown command `{cmd}`; use `bot: help` for usage information")
+
+    def handle_bot_command_help(self, event_info, bot_command):
+        """handles command 'bot: help' with a simple usage info"""
+        help_msg = "\n  **How to send commands to bot instances**"
+        help_msg += "\n  - Commands must be sent with a **new** comment (edits of existing comments are ignored)."
+        help_msg += "\n  - A comment may contain multiple commands, one per line."
+        help_msg += "\n  - Every command begins at the start of a line and has the syntax `bot: COMMAND [ARGUMENTS]*`"
+        help_msg += "\n  - Currently supported COMMANDs are: `help`, `build`, `show_config`"
+        return help_msg
+
+    def handle_bot_command_build(self, event_info, bot_command):
+        """handles command 'bot: build [ARGS*]' by parsing arguments and submitting jobs"""
+        gh = github.get_instance()
+        self.log("repository: '%s'", event_info['raw_request_body']['repository']['full_name'])
+        repo_name = event_info['raw_request_body']['repository']['full_name']
+        pr_number = event_info['raw_request_body']['issue']['number']
+        pr = gh.get_repo(repo_name).get_pull(pr_number)
+        build_msg = ''
+        if check_build_permission(pr, event_info):
+            # use filter from command
+            submitted_jobs = submit_build_jobs(pr, event_info, bot_command.action_filters)
+            if submitted_jobs is None or len(submitted_jobs) == 0:
+                build_msg = "\n  - no jobs were submitted"
+            else:
+                for job_id, issue_comment in submitted_jobs.items():
+                    build_msg += f"\n  - submitted job `{job_id}`"
+                    if issue_comment:
+                        build_msg += f", for details & status see {issue_comment.html_url}"
+        else:
+            request_body = event_info['raw_request_body']
+            sender = request_body['sender']['login']
+            build_msg = f"\n  - account `{sender}` has NO permission to submit build jobs"
+        return build_msg
+
+    def handle_bot_command_show_config(self, event_info, bot_command):
+        """handles command 'bot: show_config' by printing a list of configured build targets"""
+        self.log("processing bot command 'show_config'")
+        gh = github.get_instance()
+        repo_name = event_info['raw_request_body']['repository']['full_name']
+        pr_number = event_info['raw_request_body']['issue']['number']
+        pr = gh.get_repo(repo_name).get_pull(pr_number)
+        issue_comment = self.handle_pull_request_opened_event(event_info, pr)
+        return f"\n  - added comment {issue_comment.html_url} to show configuration"
+
     def start(self, app, port=3000):
         """starts the app and log information in the log file
 

scripts/bot-build.slurm

@@ -27,5 +27,6 @@ if [ -f bot/build.sh ]; then
     echo "bot/build.sh script found in '${PWD}', so running it!"
     bot/build.sh
 else
-    fatal_error "could not find bot/build.sh script in '${PWD}'"
+    echo "could not find bot/build.sh script in '${PWD}'" >&2
+    exit 1
 fi