feat(deadline): add ConfigureSpotEventPlugin and SpotEventPluginFleet…

… constructs (#279)

examples/deadline/All-In-AWS-Infrastructure-Basic/python/package/config.py

@@ -32,7 +32,7 @@ def __init__(self):
         # to pin to. Some examples of pinned version values are "10", "10.1", or "10.1.12"
         self.deadline_version: Optional[str] = None
 
-        # A map of regions to Deadline Client Linux AMIs.As an example, the Linux Deadline 10.1.12.1 AMI ID
+        # A map of regions to Deadline Client Linux AMIs. As an example, the Linux Deadline 10.1.12.1 AMI ID
         # from us-west-2 is filled in. It can be used as-is, added to, or replaced. Ideally the version here
         #  should match the one used for staging the render queue and usage based licensing recipes.
         self.deadline_client_linux_ami_map: Mapping[str, str] = {'us-west-2': 'ami-039f0c1faba28b015'}

examples/deadline/All-In-AWS-Infrastructure-SEP/README.md

@@ -10,7 +10,7 @@ _**Note:** This application is an illustrative example to showcase some of the c
 
 ## Architecture
 
-This sample application deploys a basic Deadline Render farm that is configured to use Deadlines [Spot Event Plugin](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html). 
+This sample application deploys a basic Deadline Render farm that is configured to use Deadline's [Spot Event Plugin](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html).
 
 ### Components
 
@@ -22,15 +22,15 @@ The Repository component contains the database and file system that store persis
 
 #### Render Queue
 
-The Render Queue component contains the fleet of [Deadline Remote Connection Server](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/remote-connection-server.html) instances behind an [Application Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/introduction.html). This acts as the central service for Deadline applications and is the only component that interacts with the Repository.  When comparing this component to the "All in AWS Infrastructure - Basic" example it has been granted additional permissions in order to use the Spot Event Plugin.    
+The Render Queue component contains the fleet of [Deadline Remote Connection Server](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/remote-connection-server.html) instances behind an [Application Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/introduction.html). This acts as the central service for Deadline applications and is the only component that interacts with the Repository. When comparing this component to the "All in AWS Infrastructure - Basic" example, it has been granted additional permissions in order to use the Spot Event Plugin.
 
 #### Spot Event Plugin Configurations
 
-The Spot Event plugin requires additional Roles for both Deadline's Resource Tracker and the Spot Workers that are created and a Security Group to allow your Spot workers the ability to access the Render Queue.
+Spot Event Plugin Configuration Setup component generates and saves the [Spot Fleet Request Configurations](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html#spot-fleet-request-configurations). The Spot Workers that are created will be configured to connect to the Render Queue. The Spot Event Plugin requires additional Role for Deadline's Resource Tracker.
 
 ## Prerequisites
 
-- The Spot Fleet Configuration requires an [Amazon Machine Image (AMI)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) with the Deadline Worker application installed. This AMI must have Deadline Installed and should be configured to connect to your repository.  For additional information on setting up your AMI please see the [Spot Event Plugin Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html).
+- The Spot Fleet Configuration requires an [Amazon Machine Image (AMI)](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) with the Deadline Worker application installed. This AMI must have Deadline Installed and should be configured to connect to your repository. For additional information on setting up your AMI please see the [Spot Event Plugin Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html).
 - You have setup and configured the AWS CLI
 - Your AWS account already has CDK bootstrapped in the desired region by running `cdk bootstrap`
 - You must have NodeJS installed on your system

examples/deadline/All-In-AWS-Infrastructure-SEP/python/README.md

@@ -13,11 +13,13 @@ These instructions assume that your working directory is `examples/deadline/All-
 ---
 
 1.  This sample app on the `mainline` branch may contain features that have not yet been officially released, and may not be available in the `aws-rfdk` package installed through pip from PyPI. To work from an example of the latest release, please switch to the `release` branch. If you would like to try out unreleased features, you can stay on `mainline` and follow the instructions for building, packing, and installing the `aws-rfdk` from your local repository.
+
 2.  Install the dependencies of the sample app:
 
     ```bash
     pip install -r requirements.txt
     ```
+
 3.  If working on the `release` branch, this step can be skipped. If working on `mainline`, navigate to the base directory where the build and packaging scripts are, then run them and install the result over top of the `aws-rfdk` version that was installed in the previous step:
     ```bash
     # Navigate to the root directory of the RFDK repository
@@ -32,33 +34,54 @@ These instructions assume that your working directory is `examples/deadline/All-
     popd
     pip install ../../../../dist/python/aws-rfdk-<version>.tar.gz
     ```
-3. Stage the Docker recipes for `RenderQueue`:
+
+4.  Change the value in the `deadline_client_linux_ami_map` variable in `package/config.py` to include the region + AMI ID mapping of your EC2 AMI(s) with Deadline Worker. You can use the following AWS CLI query to find AMI ID's:
+    ```bash
+    aws --region <region> ec2 describe-images \
+    --owners 357466774442 \
+    --filters "Name=name,Values=*Worker*" "Name=name,Values=*<version>*" \
+    --query 'Images[*].[ImageId, Name]' \
+    --output text
+    ```
+
+    And enter it into this section of `package/config.py`:
+    ```python
+    # For example, in the us-west-2 region
+    self.deadline_client_linux_ami_map: Mapping[str, str] = {
+        'us-west-2': '<your ami id>'
+    }
+    ```
+
+5. Stage the Docker recipes for `RenderQueue`:
 
     ```bash
     # Set this value to the version of RFDK your application targets
     RFDK_VERSION=<version_of_RFDK>
 
-    # Set this value to the version of AWS Thinkbox Deadline you'd like to deploy to your farm. Deadline 10.1.9 and up are supported.
+    # Set this value to the version of AWS Thinkbox Deadline you'd like to deploy to your farm. Deadline 10.1.12 and up are supported.
     RFDK_DEADLINE_VERSION=<version_of_deadline>
 
     npx --package=aws-rfdk@${RFDK_VERSION} stage-deadline --output stage ${RFDK_DEADLINE_VERSION}
     ```
-4. Deploy all the stacks in the sample app:
+
+6. Deploy all the stacks in the sample app:
 
     ```bash
     cdk deploy "*"
     ```
 
-5. Connect to your Render Farm and open up the Deadline Monitor.
+7. You can now [connect to the farm](https://docs.aws.amazon.com/rfdk/latest/guide/connecting-to-render-farm.html) and [submit rendering jobs](https://docs.aws.amazon.com/rfdk/latest/guide/first-rfdk-app.html#_optional_submit_a_job_to_the_render_farm).
+
+    **Note:** In order for the Spot Event Plugin to create a Spot Fleet Request you need to:
+    * Create the Deadline Group associated with the Spot Fleet Request Configuration. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html).
+    * Create the Deadline Pools to which the fleet Workers are added. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/pools-and-groups.html).
+    * Submit the job with the assigned Deadline Group and Deadline Pool. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/job-submitting.html#submitting-jobs).
+
+    **Note:** Disable 'Allow Workers to Perform House Cleaning If Pulse is not Running' in the 'Configure Repository Options' when using Spot Event Plugin. See [Deadline Documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html#prerequisites).
 
-6. Configure the Spot event plugin by following the directions in the [Spot Event Plugin documentation](https://docs.thinkboxsoftware.com/products/deadline/10.1/1_User%20Manual/manual/event-spot.html) with the following considerations:
+8. Once you are finished with the sample app, you can tear it down by running:
 
-    Use the default security credentials by using turning "Use Local Credentials" to False and leaving both "Access Key ID" and "Secret Access Key" blank.
-    Ensure that the Region your Spot workers will be launched in is the same region as your CDK application.
-    When Creating your Spot Fleet Requests, set the IAM instance profile to "DeadlineSpotWorkerRole" and set the security group to "DeadlineSpotSecurityGroup".
-    Configure your instances to connect to the Render Queue by either creating your AMI after launching your app and preconfiguring the AMI or by setting up a userdata in the Spot Fleet Request. (see the Spot Event Plugin documentation for additional information on configuring this connection.)
-
-7. Once you are finished with the sample app, you can tear it down by running:
+    **Note:** Any resources created by the Spot Event Plugin will not be deleted with `cdk destroy`. Make sure that all such resources (e.g. Spot Fleet Request or Fleet Instances) are cleaned up, before destroying the stacks. Disable the Spot Event Plugin by setting 'state' property to 'SpotEventPluginState.DISABLED' or via Deadline Monitor, ensure you shutdown all Pulse instances and then terminate any Spot Fleet Requests in the AWS EC2 Instance Console.
 
     ```bash
     cdk destroy "*"

examples/deadline/All-In-AWS-Infrastructure-SEP/python/package/app.py

@@ -7,14 +7,27 @@
 
 from aws_cdk.core import (
     App,
-    Environment
+    Environment,
+)
+
+from aws_cdk.aws_ec2 import (
+    MachineImage,
 )
 
 from .lib import (
     sep_stack,
 )
 
+from .config import config
+
 def main():
+    # ------------------------------
+    # Validate Config Values
+    # ------------------------------
+
+    if 'region' in config.deadline_client_linux_ami_map:
+        raise ValueError('Deadline Client Linux AMI map is required but was not specified.')
+
     # ------------------------------
     # Application
     # ------------------------------
@@ -28,11 +41,10 @@ def main():
         account=os.environ.get('CDK_DEPLOY_ACCOUNT', os.environ.get('CDK_DEFAULT_ACCOUNT')),
         region=os.environ.get('CDK_DEPLOY_REGION', os.environ.get('CDK_DEFAULT_REGION'))
     )
-    # ------------------------------
-    # Service Tier
-    # ------------------------------
+
     sep_props = sep_stack.SEPStackProps(
         docker_recipes_stage_path=os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, 'stage'),
+        worker_machine_image=MachineImage.generic_linux(config.deadline_client_linux_ami_map),
     )
     service = sep_stack.SEPStack(app, 'SEPStack', props=sep_props, env=env)
 

examples/deadline/All-In-AWS-Infrastructure-SEP/python/package/config.py

@@ -0,0 +1,23 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import (
+    List,
+    Mapping,
+    Optional,
+)
+
+class AppConfig:
+    """
+    Configuration values for the sample app.
+
+    TODO: Fill these in with your own values.
+    """
+    def __init__(self):
+        # A map of regions to Deadline Client Linux AMIs. As an example, the Linux Deadline 10.1.12.1 AMI ID
+        # from us-west-2 is filled in. It can be used as-is, added to, or replaced. Ideally the version here
+        #  should match the one used for staging the render queue and usage based licensing recipes.
+        self.deadline_client_linux_ami_map: Mapping[str, str] = {'us-west-2': 'ami-039f0c1faba28b015'}
+
+
+config: AppConfig = AppConfig()