Use --endpoint-url for s3/s3api commands.

CHANGELOG.rst

@@ -2,6 +2,13 @@
 CHANGELOG
 =========
 
+Next Release (TBD)
+
+* bugfix:``aws s3/s3api``: Fix issue where ``--endpoint-url``
+  wasn't being used for ``aws s3/s3api`` commands
+  (`issue 549 <https://github.com/aws/aws-cli/issues/549>`__)
+
+
 1.3.21
 ======
 

awscli/customizations/s3endpoint.py

@@ -0,0 +1,45 @@
+# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Disable endpoint url customizations for s3.
+
+There's a customization in botocore such that for S3 operations
+we try to fix the S3 endpoint url based on whether a bucket is
+dns compatible.  We also try to map the endpoint url to the
+standard S3 region (s3.amazonaws.com).  This normally happens
+even if a user provides an --endpoint-url (if the bucket is
+DNS compatible).
+
+This customization ensures that if a user specifies
+an --endpoint-url, then we turn off the botocore customization
+that messes with endpoint url.
+
+"""
+from functools import partial
+
+from botocore.handlers import fix_s3_host
+
+
+def register_s3_endpoint(cli):
+    handler = partial(on_top_level_args_parsed, event_handler=cli)
+    cli.register('top-level-args-parsed', handler)
+
+
+def on_top_level_args_parsed(parsed_args, event_handler, **kwargs):
+    # The fix_s3_host has logic to set the endpoint to the
+    # standard region endpoint for s3 (s3.amazonaws.com) under
+    # certain conditions.  We're making sure that if
+    # the user provides an --endpoint-url, that entire handler
+    # is disabled.
+    if parsed_args.command in ['s3', 's3api'] and \
+            parsed_args.endpoint_url is not None:
+        event_handler.unregister('before-auth.s3', fix_s3_host)

awscli/handlers.py

@@ -46,6 +46,7 @@
 from awscli.customizations.cloudsearch import initialize as cloudsearch_init
 from awscli.customizations.emr.emr import emr_initialize
 from awscli.customizations.cloudsearchdomain import register_cloudsearchdomain
+from awscli.customizations.s3endpoint import register_s3_endpoint
 
 
 def awscli_initialize(event_handlers):
@@ -94,3 +95,4 @@ def awscli_initialize(event_handlers):
     cloudsearch_init(event_handlers)
     emr_initialize(event_handlers)
     register_cloudsearchdomain(event_handlers)
+    register_s3_endpoint(event_handlers)

tests/unit/customizations/test_s3endpoint.py

@@ -0,0 +1,43 @@
+# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from awscli.testutils import unittest
+from awscli.customizations.s3endpoint import on_top_level_args_parsed
+
+from botocore.handlers import fix_s3_host
+
+import mock
+
+
+class TestS3EndpointURL(unittest.TestCase):
+    def test_endpoint_url_unregisters_fix_s3_host(self):
+        args = mock.Mock()
+        args.endpoint_url = 'http://custom/'
+        args.command = 's3'
+        event_handler = mock.Mock()
+        on_top_level_args_parsed(args, event_handler)
+        event_handler.unregister.assert_called_with('before-auth.s3', fix_s3_host)
+
+    def test_unregister_not_called_for_no_endpoint(self):
+        args = mock.Mock()
+        args.endpoint_url = None
+        event_handler = mock.Mock()
+        on_top_level_args_parsed(args, event_handler)
+        self.assertFalse(event_handler.unregister.called)
+
+    def test_endpoint_url_set_but_not_for_s3(self):
+        args = mock.Mock()
+        args.endpoint_url = 'http://custom/'
+        args.command = 'NOTS3'
+        event_handler = mock.Mock()
+        on_top_level_args_parsed(args, event_handler)
+        self.assertFalse(event_handler.unregister.called)