Add class for merging converting s3 runtime config

awscli/customizations/s3/transferconfig.py

@@ -1,4 +1,4 @@
-# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# Copyright 2013-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"). You
 # may not use this file except in compliance with the License. A copy of
@@ -10,8 +10,76 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+from awscli.customizations.s3.utils import human_readable_to_bytes
+# If the user does not specify any overrides,
+# these are the default values we use for the s3 transfer
+# commands.
+DEFAULTS = {
+    'multipart_threshold': 8 * (1024 ** 2),
+    'multipart_chunksize': 8 * (1024 ** 2),
+    'max_concurrent_requests': 10,
+    'max_queue_size': 1000,
+    # TODO: do we need this now that we have sentinels?  Need to double check.
+    # I think this was exposed before because you'd have a 0.2s delay when
+    # running some of the S3 tests.
+    'queue_timeout_wait': 0.2,
+}
 MULTI_THRESHOLD = 8 * (1024 ** 2)
 CHUNKSIZE = 7 * (1024 ** 2)
 NUM_THREADS = 10
 QUEUE_TIMEOUT_WAIT = 0.2
 MAX_QUEUE_SIZE = 1000
+
+
+class InvalidConfigError(Exception):
+    pass
+
+
+class RuntimeConfig(object):
+
+    POSITIVE_INTEGERS = ['multipart_chunksize', 'multipart_threshold',
+                         'max_concurrent_requests', 'max_queue_size']
+    HUMAN_READABLE_SIZES = ['multipart_chunksize', 'multipart_threshold']
+
+    @staticmethod
+    def defaults():
+        return DEFAULTS.copy()
+
+    def build_config(self, **kwargs):
+        """Create and convert a runtime config dictionary.
+
+        This method will merge and convert S3 runtime configuration
+        data into a single dictionary that can then be passed to classes
+        that use this runtime config.
+
+        :param kwargs:  Any key in the ``DEFAULTS`` dict.
+        :return: A dictionar of the merged and converted values.
+
+        """
+        runtime_config = DEFAULTS.copy()
+        if kwargs:
+            runtime_config.update(kwargs)
+        self._convert_human_readable_sizes(runtime_config)
+        self._validate_config(runtime_config)
+        return runtime_config
+
+    def _convert_human_readable_sizes(self, runtime_config):
+        for attr in self.HUMAN_READABLE_SIZES:
+            value = runtime_config.get(attr)
+            if value is not None and not isinstance(value, int):
+                runtime_config[attr] = human_readable_to_bytes(value)
+
+    def _validate_config(self, runtime_config):
+        for attr in self.POSITIVE_INTEGERS:
+            value = runtime_config.get(attr)
+            if value is not None:
+                try:
+                    runtime_config[attr] = int(value)
+                    if not runtime_config[attr] > 0:
+                        self._error_positive_value(attr, value)
+                except ValueError:
+                    self._error_positive_value(attr, value)
+
+    def _error_positive_value(self, name, value):
+        raise InvalidConfigError(
+            "Value for %s must be a positive integer: %s" % (name, value))

awscli/customizations/s3/utils.py

@@ -35,6 +35,13 @@
 # See: http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
 # and: http://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html
 MAX_SINGLE_UPLOAD_SIZE = 5 * (1024 ** 3)
+SIZE_SUFFIX = {
+    'kb': 1024,
+    'mb': 1024 ** 2,
+    'gb': 1024 ** 3,
+    'tb': 1024 ** 4,
+}
+
 
 
 def human_readable_size(value):
@@ -70,6 +77,28 @@ def human_readable_size(value):
             return '%.1f %s' % ((base * bytes_int / unit), suffix)
 
 
+def human_readable_to_bytes(value):
+    """Converts a human readable size to bytes.
+
+    :param value: A string such as "10MB".  If a suffix is not included,
+        then the value is assumed to be an integer representing the size
+        in bytes.
+    :returns: The converted value in bytes as an integer
+
+    """
+    suffix = value[-2:].lower()
+    has_size_identifier = (
+        len(value) >= 2 and suffix in SIZE_SUFFIX)
+    if not has_size_identifier:
+        try:
+            return int(value)
+        except ValueError:
+            raise ValueError("Invalid size value: %s" % value)
+    else:
+        multiplier = SIZE_SUFFIX[suffix]
+        return int(value[:-2]) * multiplier
+
+
 class AppendFilter(argparse.Action):
     """
     This class is used as an action when parsing the parameters.

tests/unit/customizations/s3/test_transferconfig.py

@@ -0,0 +1,61 @@
+# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from awscli.testutils import unittest
+
+from awscli.customizations.s3 import transferconfig
+
+
+class TestTransferConfig(unittest.TestCase):
+
+    def build_config_with(self, **config_from_user):
+        return transferconfig.RuntimeConfig().build_config(**config_from_user)
+
+    def test_user_provides_no_config_uses_default(self):
+        # If the user does not provide any config overrides,
+        # we should just use the default values defined in
+        # the module.
+        config = transferconfig.RuntimeConfig()
+        runtime_config = config.build_config()
+        self.assertEqual(runtime_config, transferconfig.DEFAULTS)
+
+    def test_user_provides_partial_overrides(self):
+        config_from_user = {
+            'max_concurrent_requests': '20',
+            'multipart_threshold': str(64 * (1024 ** 2)),
+        }
+        runtime_config = self.build_config_with(**config_from_user)
+        # Our overrides were accepted.
+        self.assertEqual(runtime_config['multipart_threshold'],
+                         int(config_from_user['multipart_threshold']))
+        self.assertEqual(runtime_config['max_concurrent_requests'],
+                         int(config_from_user['max_concurrent_requests']))
+        # And defaults were used for values not specified.
+        self.assertEqual(runtime_config['max_queue_size'],
+                         int(transferconfig.DEFAULTS['max_queue_size']))
+
+    def test_validates_integer_types(self):
+        with self.assertRaises(transferconfig.InvalidConfigError):
+            self.build_config_with(max_concurrent_requests="not an int")
+
+    def test_validates_positive_integers(self):
+        with self.assertRaises(transferconfig.InvalidConfigError):
+            self.build_config_with(max_concurrent_requests="-10")
+
+    def test_min_value(self):
+        with self.assertRaises(transferconfig.InvalidConfigError):
+            self.build_config_with(max_concurrent_requests="0")
+
+    def test_human_readable_sizes_converted_to_bytes(self):
+        runtime_config = self.build_config_with(multipart_threshold="10MB")
+        self.assertEqual(runtime_config['multipart_threshold'],
+                         10 * 1024 * 1024)

tests/unit/customizations/s3/test_utils.py

@@ -22,6 +22,7 @@
 from awscli.customizations.s3.utils import AppendFilter
 from awscli.customizations.s3.utils import create_warning
 from awscli.customizations.s3.utils import human_readable_size
+from awscli.customizations.s3.utils import human_readable_to_bytes
 from awscli.customizations.s3.utils import MAX_SINGLE_UPLOAD_SIZE
 
 
@@ -46,6 +47,20 @@ def _test_human_size_matches(bytes_int, expected):
     assert_equal(human_readable_size(bytes_int), expected)
 
 
+def test_convert_human_readable_to_bytes():
+    yield _test_convert_human_readable_to_bytes, "1", 1
+    yield _test_convert_human_readable_to_bytes, "1024", 1024
+    yield _test_convert_human_readable_to_bytes, "1KB", 1024
+    yield _test_convert_human_readable_to_bytes, "1kb", 1024
+    yield _test_convert_human_readable_to_bytes, "1MB", 1024 ** 2
+    yield _test_convert_human_readable_to_bytes, "1GB", 1024 ** 3
+    yield _test_convert_human_readable_to_bytes, "1TB", 1024 ** 4
+
+
+def _test_convert_human_readable_to_bytes(size_str, expected):
+    assert_equal(human_readable_to_bytes(size_str), expected)
+
+
 class AppendFilterTest(unittest.TestCase):
     def test_call(self):
         parser = argparse.ArgumentParser()