# This is a combination of 6 commits.

# This is the 1st commit message:
# This is a combination of 6 commits.
# This is the 1st commit message:
Add idempotency token autoinjection.

# The commit message boto#2 will be skipped:

#	Remove unused imports

# The commit message boto#3 will be skipped:

#	First draft of injecting idempotencyToken

# The commit message boto#4 will be skipped:

#	Register with the before-parameter-build event everywhere.

# The commit message boto#5 will be skipped:

#	Test injecting indepotency token and not replacing an existing one.

# The commit message boto#6 will be skipped:

#	Not sure how I managed to modify this.

# The commit message boto#7 will be skipped:

#	PEP8

# The commit message boto#8 will be skipped:

#	PEP8

# The commit message boto#9 will be skipped:

#	Ignore mock models from tests that do not care about the model.

# The commit message boto#10 will be skipped:

#	Remove test for members which will always be on input_shape

# The commit message boto#1 will be skipped:

#	Rewrite functional test as two functional tests

# The commit message boto#2 will be skipped:

#	Make the idempotency callback less specific/lower priority

# The commit message boto#3 will be skipped:

#	Change tests so the assert happens in the test fns instead of setup

# The commit message boto#4 will be skipped:

#	Fix tests to use MagicMock which handles iteration properly.

# The commit message boto#5 will be skipped:

#	Add better logger message

# The commit message boto#6 will be skipped:

#	Add a unit test for idempotent token injection

# The commit message boto#2 will be skipped:

#	Move idempotency check to the operation model

# The commit message boto#3 will be skipped:

#	Fix unit test to mock the new get_idempotent_members

# The commit message boto#4 will be skipped:

#	Change name and get rid of redundant property

# The commit message boto#5 will be skipped:

#	Added idempotency to the metadata

# The commit message boto#6 will be skipped:

#	Added test for a pre-provided idempotency key

.changes/next-release/feature-parameter-5401.json

@@ -0,0 +1,5 @@
+{
+  "category": "parameter", 
+  "type": "feature", 
+  "description": "Automatically inject an idempotency token into parameters marked with the idempotencyToken trait"
+}

botocore/docs/method.py

@@ -180,8 +180,8 @@ def document_model_driven_method(section, method_name, operation_model,
     context = {
         'special_shape_types': {
             'streaming_input_shape': operation_model.get_streaming_input(),
-            'streaming_output_shape': operation_model.get_streaming_output()
-        }
+            'streaming_output_shape': operation_model.get_streaming_output(),
+        },
     }
 
     if operation_model.input_shape:

botocore/docs/params.py

@@ -203,5 +203,14 @@ def _add_member_documentation(self, section, shape, name=None,
                 'param-documentation')
             documentation_section.style.indent()
             documentation_section.include_doc_string(shape.documentation)
+            self._add_special_trait_documentation(documentation_section, shape)
         end_param_section = section.add_new_section('end-param')
         end_param_section.style.new_paragraph()
+
+    def _add_special_trait_documentation(self, section, shape):
+        if 'idempotencyToken' in shape.metadata:
+            self._append_idempotency_documentation(section)
+
+    def _append_idempotency_documentation(self, section):
+        docstring = 'This field is autopopulated if not provided.'
+        section.write(docstring)

botocore/handlers.py

@@ -22,6 +22,7 @@
 import copy
 import re
 import warnings
+import uuid
 
 from botocore.compat import unquote, json, six, unquote_str, \
     ensure_bytes, get_md5, MD5_AVAILABLE
@@ -109,6 +110,14 @@ def decode_console_output(parsed, **kwargs):
             logger.debug('Error decoding base64', exc_info=True)
 
 
+def generate_idempotent_uuid(params, model, **kwargs):
+    for name in model.idempotent_members:
+        if name not in params:
+            params[name] = str(uuid.uuid4())
+            logger.debug("injecting idempotency token (%s) into param '%s'." %
+                         (params[name], name))
+
+
 def decode_quoted_jsondoc(value):
     try:
         value = json.loads(unquote(value))
@@ -594,6 +603,7 @@ def document_cloudformation_get_template_return_type(section, event_name, **kwar
         value_portion.clear_text()
         value_portion.write('{}')
 
+
 def switch_host_machinelearning(request, **kwargs):
     switch_host_with_param(request, 'PredictEndpoint')
 
@@ -764,6 +774,8 @@ def _replace_content(self, section):
     ('after-call.cloudformation.GetTemplate', json_decode_template_body),
     ('after-call.s3.GetBucketLocation', parse_get_bucket_location),
 
+    ('before-parameter-build', generate_idempotent_uuid),
+
     ('before-parameter-build.s3', validate_bucket_name),
 
     ('before-parameter-build.s3.ListObjects',
@@ -864,7 +876,7 @@ def _replace_content(self, section):
     # Cloudformation documentation customizations
     ('docs.*.cloudformation.GetTemplate.complete-section',
      document_cloudformation_get_template_return_type),
-    
+
     # UserData base64 encoding documentation customizations
     ('docs.*.ec2.RunInstances.complete-section',
      document_base64_encoding('UserData')),

botocore/model.py

@@ -49,7 +49,8 @@ class Shape(object):
     SERIALIZED_ATTRS = ['locationName', 'queryName', 'flattened', 'location',
                         'payload', 'streaming', 'timestampFormat',
                         'xmlNamespace', 'resultWrapper', 'xmlAttribute']
-    METADATA_ATTRS = ['required', 'min', 'max', 'sensitive', 'enum']
+    METADATA_ATTRS = ['required', 'min', 'max', 'sensitive', 'enum',
+                      'idempotencyToken']
     MAP_TYPE = OrderedDict
 
     def __init__(self, shape_name, shape_model, shape_resolver=None):
@@ -128,6 +129,7 @@ def metadata(self):
             * enum
             * sensitive
             * required
+            * idempotencyToken
 
         :rtype: dict
         :return: Metadata about the shape.
@@ -407,6 +409,16 @@ def output_shape(self):
         return self._service_model.resolve_shape_ref(
             self._operation_model['output'])
 
+    @CachedProperty
+    def idempotent_members(self):
+        input_shape = self.input_shape
+        if not input_shape:
+            return []
+
+        return [name for (name, shape) in input_shape.members.items()
+                if 'idempotencyToken' in shape.metadata and
+                shape.metadata['idempotencyToken']]
+
     @CachedProperty
     def has_streaming_input(self):
         return self.get_streaming_input() is not None

tests/functional/docs/test_ec2.py

@@ -30,3 +30,9 @@ def test_copy_snapshot_destination_region_is_autopopulated(self):
             service_name='ec2',
             method_name='copy_snapshot',
             param_name='DestinationRegion')
+
+    def test_idempotency_documented(self):
+        content = self.get_docstring_for_method('ec2', 'purchase_scheduled_instances')
+        # Client token should have had idempotentcy autopopulated doc appended
+        self.assert_contains_line('This field is autopopulated if not provided',
+                                  content)

tests/functional/test_ec2.py

@@ -0,0 +1,70 @@
+# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from tests import unittest
+from tests.functional.docs import BaseDocsFunctionalTest
+from botocore.stub import Stubber, StubAssertionError, ANY
+import botocore.session
+
+
+class TestIdempotencyToken(unittest.TestCase):
+    def setUp(self):
+        self.function_name = 'purchase_scheduled_instances'
+        self.region = 'us-west-2'
+        self.session = botocore.session.get_session()
+        self.client = self.session.create_client(
+            'ec2', self.region)
+        self.stubber = Stubber(self.client)
+        self.service_response = {}
+        self.params_seen = []
+
+        # Record all the parameters that get seen
+        self.client.meta.events.register_first(
+            'before-call.*.*',
+            self.collect_params,
+            unique_id='TestIdempotencyToken')
+
+    def collect_params(self, model, params, *args, **kwargs):
+        self.params_seen.extend(params['body'].keys())
+
+    def test_provided_idempotency_token(self):
+        expected_params = {
+            'PurchaseRequests': [
+                {'PurchaseToken': 'foo',
+                 'InstanceCount': 123}],
+            'ClientToken': ANY
+        }
+        self.stubber.add_response(
+            self.function_name, self.service_response, expected_params)
+
+        with self.stubber:
+            self.client.purchase_scheduled_instances(
+                PurchaseRequests=[{'PurchaseToken': 'foo',
+                                   'InstanceCount': 123}],
+                ClientToken='foobar')
+            self.assertIn('ClientToken', self.params_seen)
+
+    def test_insert_idempotency_token(self):
+        expected_params = {
+            'PurchaseRequests': [
+                {'PurchaseToken': 'foo',
+                 'InstanceCount': 123}],
+        }
+
+        self.stubber.add_response(
+            self.function_name, self.service_response, expected_params)
+
+        with self.stubber:
+            self.client.purchase_scheduled_instances(
+                PurchaseRequests=[{'PurchaseToken': 'foo',
+                                   'InstanceCount': 123}])
+            self.assertIn('ClientToken', self.params_seen)

tests/functional/test_s3.py

@@ -11,15 +11,11 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 from tests import unittest, mock, BaseSessionTest, create_session
-import os
-import nose
 from nose.tools import assert_equal
 
 import botocore.session
 from botocore.config import Config
-from botocore.compat import six
-from botocore.exceptions import ParamValidationError, ClientError
-from botocore.stub import Stubber
+from botocore.exceptions import ParamValidationError
 
 
 class TestS3BucketValidation(unittest.TestCase):

tests/unit/test_handlers.py

@@ -738,15 +738,15 @@ def test_sse_params(self):
             event = 'before-parameter-build.s3.%s' % op
             params = {'SSECustomerKey': b'bar',
                       'SSECustomerAlgorithm': 'AES256'}
-            self.session.emit(event, params=params, model=mock.Mock())
+            self.session.emit(event, params=params, model=mock.MagicMock())
             self.assertEqual(params['SSECustomerKey'], 'YmFy')
             self.assertEqual(params['SSECustomerKeyMD5'], 'Zm9v')
 
     def test_sse_params_as_str(self):
         event = 'before-parameter-build.s3.PutObject'
         params = {'SSECustomerKey': 'bar',
                   'SSECustomerAlgorithm': 'AES256'}
-        self.session.emit(event, params=params, model=mock.Mock())
+        self.session.emit(event, params=params, model=mock.MagicMock())
         self.assertEqual(params['SSECustomerKey'], 'YmFy')
         self.assertEqual(params['SSECustomerKeyMD5'], 'Zm9v')
 
@@ -755,15 +755,15 @@ def test_copy_source_sse_params(self):
             event = 'before-parameter-build.s3.%s' % op
             params = {'CopySourceSSECustomerKey': b'bar',
                       'CopySourceSSECustomerAlgorithm': 'AES256'}
-            self.session.emit(event, params=params, model=mock.Mock())
+            self.session.emit(event, params=params, model=mock.MagicMock())
             self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy')
             self.assertEqual(params['CopySourceSSECustomerKeyMD5'], 'Zm9v')
 
     def test_copy_source_sse_params_as_str(self):
         event = 'before-parameter-build.s3.CopyObject'
         params = {'CopySourceSSECustomerKey': 'bar',
                   'CopySourceSSECustomerAlgorithm': 'AES256'}
-        self.session.emit(event, params=params, model=mock.Mock())
+        self.session.emit(event, params=params, model=mock.MagicMock())
         self.assertEqual(params['CopySourceSSECustomerKey'], 'YmFy')
         self.assertEqual(params['CopySourceSSECustomerKeyMD5'], 'Zm9v')
 

tests/unit/test_idempotency.py

@@ -0,0 +1,39 @@
+# Copyright 2012-2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+from tests import unittest
+import re
+import mock
+from botocore.handlers import generate_idempotent_uuid
+
+
+class TestIdempotencyInjection(unittest.TestCase):
+    def setUp(self):
+        self.mock_model = mock.MagicMock()
+        self.mock_model.idempotent_members = ['RequiredKey']
+        self.uuid_pattern = re.compile(
+            '^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$',
+            re.I)
+
+    def test_injection(self):
+        # No parameters are provided, RequiredKey should be autofilled
+        params = {}
+        generate_idempotent_uuid(params, self.mock_model)
+        self.assertIn('RequiredKey', params)
+        self.assertIsNotNone(self.uuid_pattern.match(params['RequiredKey']))
+
+    def test_provided(self):
+        # RequiredKey is provided, should not be replaced
+        params = {'RequiredKey': 'already populated'}
+        generate_idempotent_uuid(params, self.mock_model)
+        self.assertEquals(params['RequiredKey'], 'already populated')