Add option to detect content encoding to fix gzip uploads. Resolves #451

docs/backends/amazon-S3.rst

@@ -102,6 +102,9 @@ Available are numerous settings. It should be especially noted the following:
 ``AWS_IS_GZIPPED`` (optional: default is ``False``)
     Whether or not to enable gzipping of content types specified by ``GZIP_CONTENT_TYPES``
 
+``AWS_DETECT_CONTENT_ENCODING`` (optional: default is ``False``)
+    Detect content encoding on files and set ContentEncoding headers accordingly when sending to S3.  Files gzipped before being sent to S3 will be stored uncompressed.
+
 ``GZIP_CONTENT_TYPES`` (optional: default is ``text/css``, ``text/javascript``, ``application/javascript``, ``application/x-javascript``, ``image/svg+xml``)
     When ``AWS_IS_GZIPPED`` is set to ``True`` the content types which will be gzipped
 

storages/backends/s3boto.py

@@ -199,6 +199,7 @@ class S3BotoStorage(Storage):
     secure_urls = setting('AWS_S3_SECURE_URLS', True)
     file_name_charset = setting('AWS_S3_FILE_NAME_CHARSET', 'utf-8')
     gzip = setting('AWS_IS_GZIPPED', False)
+    detect_content_encoding = setting('AWS_DETECT_CONTENT_ENCODING', False)
     preload_metadata = setting('AWS_PRELOAD_METADATA', False)
     gzip_content_types = setting('GZIP_CONTENT_TYPES', (
         'text/css',
@@ -386,7 +387,7 @@ def _save(self, name, content):
         if self.gzip and content_type in self.gzip_content_types:
             content = self._compress_content(content)
             headers.update({'Content-Encoding': 'gzip'})
-        elif encoding:
+        elif self.detect_content_encoding and encoding:
             # If the content already has a particular encoding, set it
             headers.update({'Content-Encoding': encoding})
 

storages/backends/s3boto3.py

@@ -206,6 +206,7 @@ class S3Boto3Storage(Storage):
     secure_urls = setting('AWS_S3_SECURE_URLS', True)
     file_name_charset = setting('AWS_S3_FILE_NAME_CHARSET', 'utf-8')
     gzip = setting('AWS_IS_GZIPPED', False)
+    detect_content_encoding = setting('AWS_DETECT_CONTENT_ENCODING', False)
     preload_metadata = setting('AWS_PRELOAD_METADATA', False)
     gzip_content_types = setting('GZIP_CONTENT_TYPES', (
         'text/css',
@@ -474,7 +475,7 @@ def _save(self, name, content):
         if self.gzip and content_type in self.gzip_content_types:
             content = self._compress_content(content)
             parameters.update({'ContentEncoding': 'gzip'})
-        elif encoding:
+        elif self.detect_content_encoding and encoding:
             # If the content already has a particular encoding, set it
             parameters.update({'ContentEncoding': encoding})
 

tests/test_s3boto.py

@@ -4,6 +4,7 @@
     import mock
 
 import datetime
+import zlib
 
 from boto.exception import S3ResponseError
 from boto.s3.key import Key
@@ -92,7 +93,29 @@ def test_storage_save_gzipped(self):
         Test saving a gzipped file
         """
         name = 'test_storage_save.gz'
-        content = ContentFile("I am gzip'd")
+        compressed_data = zlib.compress("I am gzip'd".encode('utf8'))
+        content = ContentFile(compressed_data)
+        self.storage.detect_content_encoding = False
+        self.storage.save(name, content)
+        key = self.storage.bucket.get_key.return_value
+        key.set_metadata.assert_called_with('Content-Type',
+                                            'application/octet-stream')
+        key.set_contents_from_file.assert_called_with(
+            content,
+            headers={'Content-Type': 'application/octet-stream'},
+            policy=self.storage.default_acl,
+            reduced_redundancy=self.storage.reduced_redundancy,
+            rewind=True,
+        )
+
+    def test_storage_save_gzip_encoded(self):
+        """
+        Test saving a gzipped file
+        """
+        name = 'test_storage_save.gz'
+        compressed_data = zlib.compress("I am gzip'd".encode('utf8'))
+        content = ContentFile(compressed_data)
+        self.storage.detect_content_encoding = True
         self.storage.save(name, content)
         key = self.storage.bucket.get_key.return_value
         key.set_metadata.assert_called_with('Content-Type',

tests/test_s3boto3.py

@@ -5,6 +5,7 @@
 import pickle
 import threading
 import warnings
+import zlib
 from datetime import datetime
 from unittest import skipIf
 
@@ -166,7 +167,27 @@ def test_storage_save_gzipped(self):
         Test saving a gzipped file
         """
         name = 'test_storage_save.gz'
-        content = ContentFile("I am gzip'd")
+        compressed_data = zlib.compress("I am gzip'd".encode('utf8'))
+        content = ContentFile(compressed_data)
+        self.storage.detect_content_encoding = False
+        self.storage.save(name, content)
+        obj = self.storage.bucket.Object.return_value
+        obj.upload_fileobj.assert_called_with(
+            content.file,
+            ExtraArgs={
+                'ContentType': 'application/octet-stream',
+                'ACL': self.storage.default_acl,
+            }
+        )
+
+    def test_storage_save_gzip_encoded(self):
+        """
+        Test saving a gzipped file
+        """
+        name = 'test_storage_save.gz'
+        compressed_data = zlib.compress("I am gzip'd".encode('utf8'))
+        content = ContentFile(compressed_data)
+        self.storage.detect_content_encoding = True
         self.storage.save(name, content)
         obj = self.storage.bucket.Object.return_value
         obj.upload_fileobj.assert_called_with(