Emr configurations values: also allow AWS helper functions (cloudtool…

…s#516) * Added parameter to dynamically choose between spot and on-demand instances * Fixed EMR configuration properties validator: both strings and helper functions are allowed by cloudformation * Prevent lines with more than 79 characters to make Travis CI happy * Fixed EMR configuration properties validator: both strings and helper functions are allowed by cloudformation
amosshapira · Oct 24, 2016 · 9e3edc0 · 9e3edc0
1 parent 3c8c789
commit 9e3edc0
Show file tree

Hide file tree

Showing 3 changed files with 80 additions and 11 deletions.
diff --git a/examples/EMR_Cluster.py b/examples/EMR_Cluster.py
@@ -1,5 +1,5 @@
-from troposphere import Parameter, Ref, Template, Tags
-from troposphere.constants import KEY_PAIR_NAME, SUBNET_ID, M4_LARGE
+from troposphere import Parameter, Ref, Template, Tags, If, Equals, Not, Join
+from troposphere.constants import KEY_PAIR_NAME, SUBNET_ID, M4_LARGE, NUMBER
 import troposphere.emr as emr
 import troposphere.iam as iam
 
@@ -22,6 +22,23 @@
     Type=SUBNET_ID
 ))
 
+spot = template.add_parameter(Parameter(
+    "SpotPrice",
+    Description="Spot price (or use 0 for 'on demand' instance)",
+    Type=NUMBER,
+    Default="0.1"
+))
+
+withSpotPrice = "WithSpotPrice"
+template.add_condition(withSpotPrice, Not(Equals(Ref(spot), "0")))
+
+gcTimeRatio = template.add_parameter(Parameter(
+    "GcTimeRatioValue",
+    Description="Hadoop name node garbage collector time ratio",
+    Type=NUMBER,
+    Default="19"
+))
+
 # IAM roles required by EMR
 
 emr_service_role = template.add_resource(iam.Role(
@@ -100,7 +117,8 @@
                     Classification="export",
                     ConfigurationProperties={
                         "HADOOP_DATANODE_HEAPSIZE": "2048",
-                        "HADOOP_NAMENODE_OPTS": "-XX:GCTimeRatio=19"
+                        "HADOOP_NAMENODE_OPTS": Join("", ["-XX:GCTimeRatio=",
+                                                          Ref(gcTimeRatio)])
                     }
                 )
             ]
@@ -119,7 +137,8 @@
         ),
         CoreInstanceGroup=emr.InstanceGroupConfigProperty(
             Name="Core Instance",
-            BidPrice="0.1",
+            BidPrice=If(withSpotPrice, Ref(spot), Ref("AWS::NoValue")),
+            Market=If(withSpotPrice, "SPOT", "ON_DEMAND"),
             EbsConfiguration=emr.EbsConfiguration(
                 EbsBlockDeviceConfigs=[
                     emr.EbsBlockDeviceConfigs(
@@ -134,7 +153,6 @@
             ),
             InstanceCount="1",
             InstanceType=M4_LARGE,
-            Market="SPOT"
         )
     ),
     Applications=[

diff --git a/tests/examples_output/EMR_Cluster.template b/tests/examples_output/EMR_Cluster.template
@@ -1,10 +1,34 @@
 {
+    "Conditions": {
+        "WithSpotPrice": {
+            "Fn::Not": [
+                {
+                    "Fn::Equals": [
+                        {
+                            "Ref": "SpotPrice"
+                        },
+                        "0"
+                    ]
+                }
+            ]
+        }
+    },
     "Description": "Sample CloudFormation template for creating an EMR cluster",
     "Parameters": {
+        "GcTimeRatioValue": {
+            "Default": "19",
+            "Description": "Hadoop name node garbage collector time ratio",
+            "Type": "Number"
+        },
         "KeyName": {
             "Description": "Name of an existing EC2 KeyPair to enable SSH to the instances",
             "Type": "AWS::EC2::KeyPair::KeyName"
         },
+        "SpotPrice": {
+            "Default": "0.1",
+            "Description": "Spot price (or use 0 for 'on demand' instance)",
+            "Type": "Number"
+        },
         "Subnet": {
             "Description": "Subnet ID for creating the EMR cluster",
             "Type": "AWS::EC2::Subnet::Id"
@@ -97,15 +121,35 @@
                                 "Classification": "export",
                                 "ConfigurationProperties": {
                                     "HADOOP_DATANODE_HEAPSIZE": "2048",
-                                    "HADOOP_NAMENODE_OPTS": "-XX:GCTimeRatio=19"
+                                    "HADOOP_NAMENODE_OPTS": {
+                                        "Fn::Join": [
+                                            "",
+                                            [
+                                                "-XX:GCTimeRatio=",
+                                                {
+                                                    "Ref": "GcTimeRatioValue"
+                                                }
+                                            ]
+                                        ]
+                                    }
                                 }
                             }
                         ]
                     }
                 ],
                 "Instances": {
                     "CoreInstanceGroup": {
-                        "BidPrice": "0.1",
+                        "BidPrice": {
+                            "Fn::If": [
+                                "WithSpotPrice",
+                                {
+                                    "Ref": "SpotPrice"
+                                },
+                                {
+                                    "Ref": "AWS::NoValue"
+                                }
+                            ]
+                        },
                         "EbsConfiguration": {
                             "EbsBlockDeviceConfigs": [
                                 {
@@ -120,7 +164,13 @@
                         },
                         "InstanceCount": "1",
                         "InstanceType": "m4.large",
-                        "Market": "SPOT",
+                        "Market": {
+                            "Fn::If": [
+                                "WithSpotPrice",
+                                "SPOT",
+                                "ON_DEMAND"
+                            ]
+                        },
                         "Name": "Core Instance"
                     },
                     "Ec2KeyName": {

diff --git a/troposphere/emr.py b/troposphere/emr.py
@@ -3,7 +3,7 @@
 #
 # See LICENSE file for full license.
 
-from . import AWSObject, AWSProperty
+from . import AWSObject, AWSProperty, AWSHelperFn
 from .validators import (boolean, integer, positive_integer)
 
 
@@ -65,8 +65,9 @@ def properties_validator(xs):
     for k, v in xs.iteritems():
         if not isinstance(k, basestring):
             raise ValueError('ConfigurationProperties keys must be strings')
-        if not isinstance(v, basestring):
-            raise ValueError('ConfigurationProperties values must be strings')
+        if not isinstance(v, basestring) and not isinstance(v, AWSHelperFn):
+            raise ValueError('ConfigurationProperties values must be strings'
+                             ' or helper functions')
 
     return xs