From e27bbc629582068709af5a0e52553e13f127b0dc Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Wed, 2 Jul 2014 16:07:14 -0700 Subject: [PATCH 1/2] Disable fix_s3_host when --endpoint-url is given Otherwise the --endpoint-url may not be honored if the bucket is a dns compatible bucket. --- awscli/customizations/s3endpoint.py | 45 ++++++++++++++++++++ awscli/handlers.py | 2 + tests/unit/customizations/test_s3endpoint.py | 43 +++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 awscli/customizations/s3endpoint.py create mode 100644 tests/unit/customizations/test_s3endpoint.py diff --git a/awscli/customizations/s3endpoint.py b/awscli/customizations/s3endpoint.py new file mode 100644 index 000000000000..e51a85b7e939 --- /dev/null +++ b/awscli/customizations/s3endpoint.py @@ -0,0 +1,45 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Disable endpoint url customizations for s3. + +There's a customization in botocore such that for S3 operations +we try to fix the S3 endpoint url based on whether a bucket is +dns compatible. We also try to map the endpoint url to the +standard S3 region (s3.amazonaws.com). This normally happens +even if a user provides an --endpoint-url (if the bucket is +DNS compatible). + +This customization ensures that if a user specifies +an --endpoint-url, then we turn off the botocore customization +that messes with endpoint url. + +""" +from functools import partial + +from botocore.handlers import fix_s3_host + + +def register_s3_endpoint(cli): + handler = partial(on_top_level_args_parsed, event_handler=cli) + cli.register('top-level-args-parsed', handler) + + +def on_top_level_args_parsed(parsed_args, event_handler, **kwargs): + # The fix_s3_host has logic to set the endpoint to the + # standard region endpoint for s3 (s3.amazonaws.com) under + # certain conditions. We're making sure that if + # the user provides an --endpoint-url, that entire handler + # is disabled. + if parsed_args.command in ['s3', 's3api'] and \ + parsed_args.endpoint_url is not None: + event_handler.unregister('before-auth.s3', fix_s3_host) diff --git a/awscli/handlers.py b/awscli/handlers.py index 2badab18244f..6d99892b9d9e 100644 --- a/awscli/handlers.py +++ b/awscli/handlers.py @@ -46,6 +46,7 @@ from awscli.customizations.cloudsearch import initialize as cloudsearch_init from awscli.customizations.emr.emr import emr_initialize from awscli.customizations.cloudsearchdomain import register_cloudsearchdomain +from awscli.customizations.s3endpoint import register_s3_endpoint def awscli_initialize(event_handlers): @@ -94,3 +95,4 @@ def awscli_initialize(event_handlers): cloudsearch_init(event_handlers) emr_initialize(event_handlers) register_cloudsearchdomain(event_handlers) + register_s3_endpoint(event_handlers) diff --git a/tests/unit/customizations/test_s3endpoint.py b/tests/unit/customizations/test_s3endpoint.py new file mode 100644 index 000000000000..1f83f2e5b129 --- /dev/null +++ b/tests/unit/customizations/test_s3endpoint.py @@ -0,0 +1,43 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from awscli.testutils import unittest +from awscli.customizations.s3endpoint import on_top_level_args_parsed + +from botocore.handlers import fix_s3_host + +import mock + + +class TestS3EndpointURL(unittest.TestCase): + def test_endpoint_url_unregisters_fix_s3_host(self): + args = mock.Mock() + args.endpoint_url = 'http://custom/' + args.command = 's3' + event_handler = mock.Mock() + on_top_level_args_parsed(args, event_handler) + event_handler.unregister.assert_called_with('before-auth.s3', fix_s3_host) + + def test_unregister_not_called_for_no_endpoint(self): + args = mock.Mock() + args.endpoint_url = None + event_handler = mock.Mock() + on_top_level_args_parsed(args, event_handler) + self.assertFalse(event_handler.unregister.called) + + def test_endpoint_url_set_but_not_for_s3(self): + args = mock.Mock() + args.endpoint_url = 'http://custom/' + args.command = 'NOTS3' + event_handler = mock.Mock() + on_top_level_args_parsed(args, event_handler) + self.assertFalse(event_handler.unregister.called) From 9352c6b7edc4c960d7b779c12485de84b447a880 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Wed, 2 Jul 2014 16:20:40 -0700 Subject: [PATCH 2/2] Update changelog with fix for #549 --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dad42f72b3f9..55a08ed0afd4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,13 @@ CHANGELOG ========= +Next Release (TBD) + +* bugfix:``aws s3/s3api``: Fix issue where ``--endpoint-url`` + wasn't being used for ``aws s3/s3api`` commands + (`issue 549 `__) + + 1.3.21 ======