feat: linux ec2-based tests test installer-based user creation

Problem:

The linux ec2-based tests are currently structured to create the
worker-agent user during instance startup using ec2-userdata scripting.
The worker agent installer contains code that will create the agent-user
if it does not exist. The current test setup prevents writing tests that
test that functionality

Solution:

Refactor the scripting that is materialized in the PosixInstanceWorker.
Now, user-creation-wise the ec2-userdata only creates the job users and
their groups. The 'worker configuration' scripting is now responsible
for letting the install script create the worker agent user, and adding
it to the required groups & sudoers rules.

Signed-off-by: Daniel Neilson <53624638+ddneilson@users.noreply.github.com>

src/deadline_test_fixtures/deadline/worker.py

@@ -583,12 +583,15 @@ class PosixInstanceWorker(EC2InstanceWorker):
     def ssm_document_name(self) -> str:
         return "AWS-RunShellScript"
 
+    def send_command(self, command: str) -> CommandResult:
+        return super().send_command("set -eou pipefail; " + command)
+
     def _start_worker_agent(self) -> None:
         assert self.instance_id
         LOG.info(f"Sending SSM command to configure Worker agent on instance {self.instance_id}")
 
         cmd_result = self.send_command(
-            f"cd /home/{self.configuration.agent_user}; . .venv/bin/activate; AWS_DEFAULT_REGION={self.configuration.region} {self.configure_worker_command(config=self.configuration)}"
+           self.configure_worker_command(config=self.configuration)
         )
         assert cmd_result.exit_code == 0, f"Failed to configure Worker agent: {cmd_result}"
         LOG.info("Successfully configured Worker agent")
@@ -617,6 +620,8 @@ def configure_worker_command(
         """Get the command to configure the Worker. This must be run as root."""
 
         cmds = [
+            "source /opt/deadline/worker/bin/activate",
+            "AWS_DEFAULT_REGION={self.configuration.region}",
             config.worker_agent_install.install_command_for_linux,
             *(config.pre_install_commands or []),
             # fmt: off
@@ -633,8 +638,22 @@ def configure_worker_command(
                 + f"{'--start ' if config.start_service else ''}"
             ),
             # fmt: on
+            f"runuser --login {self.configuration.agent_user} --command 'echo \"source /opt/deadline/worker/bin/activate\" >> $HOME/.bashrc'",
         ]
 
+        for job_user in self.configuration.job_users:
+            cmds.append(f"usermod -a -G {job_user.group} {self.configuration.agent_user}")
+
+        sudoer_rule_users = ",".join(
+            [
+                self.configuration.agent_user,
+                *[job_user.user for job_user in self.configuration.job_users],
+            ]
+        )
+        cmds.append(
+            f'echo "{self.configuration.agent_user} ALL=({sudoer_rule_users}) NOPASSWD: ALL" > /etc/sudoers.d/{self.configuration.agent_user}'
+        )
+
         if config.service_model_path:
             cmds.append(
                 f"runuser -l {config.agent_user} -s /bin/bash -c 'aws configure add-model --service-model file://{config.service_model_path}'"
@@ -667,7 +686,7 @@ def userdata(self, s3_files) -> str:
         if s3_files:
             copy_s3_command = " && ".join(
                 [
-                    f"aws s3 cp {s3_uri} {dst} && chown {self.configuration.agent_user} {dst}"
+                    f"aws s3 cp {s3_uri} {dst} && chmod o+rx {dst}"
                     for s3_uri, dst in s3_files
                 ]
             )
@@ -676,29 +695,18 @@ def userdata(self, s3_files) -> str:
             job_users_cmds.append(
                 f"useradd --create-home --system --shell=/bin/bash --groups={self.configuration.job_user_group} -g {job_user.group} {job_user.user}"
             )
-            job_users_cmds.append(f"usermod -a -G {job_user.group} {self.configuration.agent_user}")
-
-        sudoer_rule_users = ",".join(
-            [
-                self.configuration.agent_user,
-                *[job_user.user for job_user in self.configuration.job_users],
-            ]
-        )
-        job_users_cmds.append(
-            f'echo "{self.configuration.agent_user} ALL=({sudoer_rule_users}) NOPASSWD: ALL" > /etc/sudoers.d/{self.configuration.agent_user}'
-        )
 
         configure_job_users = "\n".join(job_users_cmds)
 
         userdata = f"""#!/bin/bash
         # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
         set -x
         groupadd --system {self.configuration.job_user_group}
-        useradd --create-home --system --shell=/bin/bash --groups={self.configuration.job_user_group} {self.configuration.agent_user}
         {configure_job_users}
         {copy_s3_command}
 
-        runuser --login {self.configuration.agent_user} --command 'python3 -m venv $HOME/.venv && echo ". $HOME/.venv/bin/activate" >> $HOME/.bashrc'
+        mkdir /opt/deadline
+        python3 -m venv /opt/deadline/worker
         """
 
         return userdata

test/unit/deadline/test_worker.py

@@ -288,7 +288,7 @@ def test_sends_command(self, worker: PosixInstanceWorker) -> None:
             send_command_spy.assert_called_once_with(
                 InstanceIds=[worker.instance_id],
                 DocumentName="AWS-RunShellScript",
-                Parameters={"commands": [cmd]},
+                Parameters={"commands": [ "set -eou pipefail; " + cmd]},
             )
 
         def test_retries_when_instance_not_ready(self, worker: PosixInstanceWorker) -> None:
@@ -323,7 +323,7 @@ def side_effect(*args, **kwargs):
                     call(
                         InstanceIds=[worker.instance_id],
                         DocumentName="AWS-RunShellScript",
-                        Parameters={"commands": [cmd]},
+                        Parameters={"commands": [ "set -eou pipefail; "+cmd]},
                     )
                 ]
                 * 2
@@ -351,7 +351,7 @@ def test_raises_any_other_error(self, worker: PosixInstanceWorker) -> None:
             mock_send_command.assert_called_once_with(
                 InstanceIds=[worker.instance_id],
                 DocumentName="AWS-RunShellScript",
-                Parameters={"commands": [cmd]},
+                Parameters={"commands": [ "set -eou pipefail; " + cmd]},
             )
 
     @pytest.mark.parametrize(