Skip to content

Commit

Permalink
Fix s3 streaming test for memory utilization
Browse files Browse the repository at this point in the history
We need to use the specified values in this class
instead of the .defaults() value.
  • Loading branch information
jamesls committed Feb 4, 2015
1 parent db24830 commit d83bfc4
Showing 1 changed file with 12 additions and 1 deletion.
13 changes: 12 additions & 1 deletion awscli/customizations/s3/s3handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -355,12 +355,23 @@ class S3StreamHandler(S3Handler):
involves a stream since the logic is different when uploading and
downloading streams.
"""

# This ensures that the number of multipart chunks waiting in the
# executor queue and in the threads is limited.
MAX_EXECUTOR_QUEUE_SIZE = 2
EXECUTOR_NUM_THREADS = 6

def __init__(self, session, params, result_queue=None,
runtime_config=None):
if runtime_config is None:
# Rather than using the .defaults(), streaming
# has different default values so that it does not
# consume large amounts of memory.
runtime_config = RuntimeConfig().build_config(
max_queue_size=self.MAX_EXECUTOR_QUEUE_SIZE,
max_concurrent_requests=self.EXECUTOR_NUM_THREADS)
super(S3StreamHandler, self).__init__(session, params, result_queue,
runtime_config)

def _enqueue_tasks(self, files):
total_files = 0
total_parts = 0
Expand Down

0 comments on commit d83bfc4

Please sign in to comment.