From d83bfc483e4b409cbfd568381a816a17c0bc5046 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Wed, 4 Feb 2015 14:12:24 -0800 Subject: [PATCH] Fix s3 streaming test for memory utilization We need to use the specified values in this class instead of the .defaults() value. --- awscli/customizations/s3/s3handler.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/awscli/customizations/s3/s3handler.py b/awscli/customizations/s3/s3handler.py index 2e9420f45817..583502b74af6 100644 --- a/awscli/customizations/s3/s3handler.py +++ b/awscli/customizations/s3/s3handler.py @@ -355,12 +355,23 @@ class S3StreamHandler(S3Handler): involves a stream since the logic is different when uploading and downloading streams. """ - # This ensures that the number of multipart chunks waiting in the # executor queue and in the threads is limited. MAX_EXECUTOR_QUEUE_SIZE = 2 EXECUTOR_NUM_THREADS = 6 + def __init__(self, session, params, result_queue=None, + runtime_config=None): + if runtime_config is None: + # Rather than using the .defaults(), streaming + # has different default values so that it does not + # consume large amounts of memory. + runtime_config = RuntimeConfig().build_config( + max_queue_size=self.MAX_EXECUTOR_QUEUE_SIZE, + max_concurrent_requests=self.EXECUTOR_NUM_THREADS) + super(S3StreamHandler, self).__init__(session, params, result_queue, + runtime_config) + def _enqueue_tasks(self, files): total_files = 0 total_parts = 0