Skip to content

Commit

Permalink
Add max_bandwidth to TransferConfig
Browse files Browse the repository at this point in the history
  • Loading branch information
nateprewitt committed Oct 27, 2021
1 parent 5dd6242 commit 5ffd699
Show file tree
Hide file tree
Showing 3 changed files with 42 additions and 8 deletions.
5 changes: 5 additions & 0 deletions .changes/next-release/enchancement-s3-6820.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"type": "enchancement",
"category": "``s3``",
"description": "TransferConfig now supports the `max_bandwidth` argument."
}
24 changes: 16 additions & 8 deletions boto3/s3/transfer.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,14 +166,17 @@ class TransferConfig(S3TransferConfig):
'max_io_queue': 'max_io_queue_size'
}

def __init__(self,
multipart_threshold=8 * MB,
max_concurrency=10,
multipart_chunksize=8 * MB,
num_download_attempts=5,
max_io_queue=100,
io_chunksize=256 * KB,
use_threads=True):
def __init__(
self,
multipart_threshold=8 * MB,
max_concurrency=10,
multipart_chunksize=8 * MB,
num_download_attempts=5,
max_io_queue=100,
io_chunksize=256 * KB,
use_threads=True,
max_bandwidth=None,
):
"""Configuration object for managed S3 transfers
:param multipart_threshold: The transfer size threshold for which
Expand Down Expand Up @@ -209,6 +212,10 @@ def __init__(self,
:param use_threads: If True, threads will be used when performing
S3 transfers. If False, no threads will be used in
performing transfers: all logic will be ran in the main thread.
:param max_bandwidth: The maximum bandwidth that will be consumed
in uploading and downloading file content. The value is an integer
in terms of bytes per second.
"""
super(TransferConfig, self).__init__(
multipart_threshold=multipart_threshold,
Expand All @@ -217,6 +224,7 @@ def __init__(self,
num_download_attempts=num_download_attempts,
max_io_queue_size=max_io_queue,
io_chunksize=io_chunksize,
max_bandwidth=max_bandwidth,
)
# Some of the argument names are not the same as the inherited
# S3TransferConfig so we add aliases so you can still access the
Expand Down
21 changes: 21 additions & 0 deletions tests/unit/s3/test_transfer.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from boto3.s3.transfer import S3Transfer
from boto3.s3.transfer import OSUtils, TransferConfig, ProgressCallbackInvoker
from boto3.s3.transfer import ClientError, S3TransferRetriesExceededError
from boto3.s3.transfer import KB, MB


class TestCreateTransferManager(unittest.TestCase):
Expand Down Expand Up @@ -83,6 +84,26 @@ def test_alias_max_io_queue(self):
self.assert_value_of_actual_and_alias(
config, 'max_io_queue_size', 'max_io_queue', new_value)

def test_transferconfig_parameters(self):
config = TransferConfig(
multipart_threshold=8 * MB,
max_concurrency=10,
multipart_chunksize=8 * MB,
num_download_attempts=5,
max_io_queue=100,
io_chunksize=256 * KB,
use_threads=True,
max_bandwidth=1024 * KB,
)
assert config.multipart_threshold == 8 * MB
assert config.multipart_chunksize == 8 * MB
assert config.max_request_concurrency == 10
assert config.num_download_attempts == 5
assert config.max_io_queue_size == 100
assert config.io_chunksize == 256 * KB
assert config.use_threads is True
assert config.max_bandwidth == 1024 * KB


class TestProgressCallbackInvoker(unittest.TestCase):
def test_on_progress(self):
Expand Down

0 comments on commit 5ffd699

Please sign in to comment.