[s3] Add AWS_S3_USE_THREADS configuration variable (#1112)

jschneier · Mar 12, 2022 · 0ceb9e0 · 0ceb9e0
1 parent 8bfb717
commit 0ceb9e0
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 11 deletions.
diff --git a/AUTHORS b/AUTHORS
@@ -47,6 +47,7 @@ By order of apparition, thanks:
     * Jonathan Ehwald
     * Dan Hook
     * François Freitag (S3)
+    * Uxío Fuentefría (S3)
 
 
 Extra thanks to Marty for adding this in Django,

diff --git a/storages/backends/s3boto3.py b/storages/backends/s3boto3.py
@@ -23,6 +23,7 @@
 
 try:
     import boto3.session
+    from boto3.s3.transfer import TransferConfig
     from botocore.client import Config
     from botocore.exceptions import ClientError
     from botocore.signers import CloudFrontSigner
@@ -133,7 +134,7 @@ def _get_file(self):
             )
             if 'r' in self._mode:
                 self._is_dirty = False
-                self.obj.download_fileobj(self._file)
+                self.obj.download_fileobj(self._file, Config=self._storage._transfer_config)
                 self._file.seek(0)
             if self._storage.gzip and self.obj.content_encoding == 'gzip':
                 self._file = self._decompress_file(mode=self._mode, file=self._file)
@@ -272,6 +273,7 @@ def __init__(self, **settings):
                 signature_version=self.signature_version,
                 proxies=self.proxies,
             )
+        self._transfer_config = TransferConfig(use_threads=self.use_threads)
 
     def get_cloudfront_signer(self, key_id, key):
         return _cloud_front_signer_from_pem(key_id, key)
@@ -331,6 +333,7 @@ def get_default_settings(self):
             'verify': setting('AWS_S3_VERIFY', None),
             'max_memory_size': setting('AWS_S3_MAX_MEMORY_SIZE', 0),
             'default_acl': setting('AWS_DEFAULT_ACL', None),
+            'use_threads': setting('AWS_S3_USE_THREADS', True),
         }
 
     def __getstate__(self):
@@ -452,7 +455,7 @@ def _save(self, name, content):
             params['ContentEncoding'] = 'gzip'
 
         obj = self.bucket.Object(name)
-        obj.upload_fileobj(content, ExtraArgs=params)
+        obj.upload_fileobj(content, ExtraArgs=params, Config=self._transfer_config)
         return cleaned_name
 
     def delete(self, name):

diff --git a/tests/test_s3boto3.py b/tests/test_s3boto3.py
@@ -121,7 +121,8 @@ def test_storage_save(self):
             content,
             ExtraArgs={
                 'ContentType': 'text/plain',
-            }
+            },
+            Config=self.storage._transfer_config
         )
 
     def test_storage_save_non_seekable(self):
@@ -138,7 +139,8 @@ def test_storage_save_non_seekable(self):
             content,
             ExtraArgs={
                 'ContentType': 'text/plain',
-            }
+            },
+            Config=self.storage._transfer_config
         )
 
     def test_storage_save_with_default_acl(self):
@@ -157,7 +159,8 @@ def test_storage_save_with_default_acl(self):
             ExtraArgs={
                 'ContentType': 'text/plain',
                 'ACL': 'private',
-            }
+            },
+            Config=self.storage._transfer_config
         )
 
     def test_storage_object_parameters_not_overwritten_by_default(self):
@@ -177,7 +180,8 @@ def test_storage_object_parameters_not_overwritten_by_default(self):
             ExtraArgs={
                 'ContentType': 'text/plain',
                 'ACL': 'private',
-            }
+            },
+            Config=self.storage._transfer_config
         )
 
     def test_content_type(self):
@@ -195,7 +199,8 @@ def test_content_type(self):
             content,
             ExtraArgs={
                 'ContentType': 'image/jpeg',
-            }
+            },
+            Config=self.storage._transfer_config
         )
 
     def test_storage_save_gzipped(self):
@@ -211,7 +216,8 @@ def test_storage_save_gzipped(self):
             ExtraArgs={
                 'ContentType': 'application/octet-stream',
                 'ContentEncoding': 'gzip',
-            }
+            },
+            Config=self.storage._transfer_config
         )
 
     def test_storage_save_gzipped_non_seekable(self):
@@ -227,7 +233,8 @@ def test_storage_save_gzipped_non_seekable(self):
             ExtraArgs={
                 'ContentType': 'application/octet-stream',
                 'ContentEncoding': 'gzip',
-            }
+            },
+            Config=self.storage._transfer_config
         )
 
     def test_storage_save_gzip(self):
@@ -244,7 +251,8 @@ def test_storage_save_gzip(self):
             ExtraArgs={
                 'ContentType': 'text/css',
                 'ContentEncoding': 'gzip',
-            }
+            },
+            Config=self.storage._transfer_config
         )
         args, kwargs = obj.upload_fileobj.call_args
         content = args[0]
@@ -271,7 +279,8 @@ def test_storage_save_gzip_twice(self):
             ExtraArgs={
                 'ContentType': 'text/css',
                 'ContentEncoding': 'gzip',
-            }
+            },
+            Config=self.storage._transfer_config
         )
         args, kwargs = obj.upload_fileobj.call_args
         content = args[0]