Merge pull request #182 from ZuluPro/clean_storage

Removed storages and adapt code

dbbackup/management/commands/_base.py

@@ -9,7 +9,7 @@
 from django.core.management.base import BaseCommand, LabelCommand, CommandError
 from django.utils import six
 
-from ...storage.base import StorageError
+from ...storage import StorageError
 
 input = raw_input if six.PY2 else input  # @ReservedAssignment
 
@@ -43,9 +43,7 @@ def read_from_storage(self, path):
         return self.storage.read_file(path)
 
     def write_to_storage(self, file, path):
-        self.logger.info("Writing file to %s: %s, filename: %s",
-                         self.storage.name, self.storage.backup_dir,
-                         path)
+        self.logger.info("Writing file to %s", path)
         self.storage.write_file(file, path)
 
     def read_local_file(self, path):
@@ -88,8 +86,8 @@ def _cleanup_old_backups(self):
         DBBACKUP_CLEANUP_KEEP and any backups that occur on first of the month.
         """
         # database = self.database if self.content_type == 'db' else None
-        file_list = self.storage.clean_old_backups(encrypted=self.encrypt,
-                                                   compressed=self.compress,
-                                                   content_type=self.content_type)
-                                                   # TODO: Make better filter
-                                                   # database=database)
+        self.storage.clean_old_backups(encrypted=self.encrypt,
+                                       compressed=self.compress,
+                                       content_type=self.content_type)
+                                       # TODO: Make better filter
+                                       # database=database)

dbbackup/management/commands/dbbackup.py

@@ -10,7 +10,7 @@
 
 from ._base import BaseDbBackupCommand
 from ...db.base import get_connector
-from ...storage.base import BaseStorage, StorageError
+from ...storage import get_storage, StorageError
 from ... import utils, settings as dbbackup_settings
 
 
@@ -48,7 +48,7 @@ def handle(self, **options):
         self.encrypt = options.get('encrypt')
         self.filename = options.get('output_filename')
         self.path = options.get('output_path')
-        self.storage = BaseStorage.storage_factory()
+        self.storage = get_storage()
         database_keys = (self.database,) if self.database else dbbackup_settings.DATABASES
         for database_key in database_keys:
             self.connector = get_connector(database_key)
@@ -78,6 +78,7 @@ def _save_new_backup(self, database):
         if not self.quiet:
             self.logger.info("Backup size: %s", utils.handle_size(outputfile))
         # Store backup
+        outputfile.seek(0)
         if self.path is None:
             self.write_to_storage(outputfile, filename)
         else:

dbbackup/management/commands/dbrestore.py

@@ -13,7 +13,7 @@
 from ._base import BaseDbBackupCommand
 from ... import utils
 from ...db.base import get_connector
-from ...storage.base import BaseStorage, StorageError
+from ...storage import get_storage, StorageError
 
 
 class Command(BaseDbBackupCommand):
@@ -47,7 +47,7 @@ def handle(self, *args, **options):
             self.passphrase = options.get('passphrase')
             self.interactive = options.get('interactive')
             self.database = self._get_database(options)
-            self.storage = BaseStorage.storage_factory()
+            self.storage = get_storage()
             self._restore_backup()
         except StorageError as err:
             raise CommandError(err)

dbbackup/management/commands/listbackups.py

@@ -6,7 +6,7 @@
 from optparse import make_option
 from ... import utils
 from ._base import BaseDbBackupCommand
-from ...storage.base import BaseStorage, StorageError
+from ...storage import get_storage
 
 ROW_TEMPLATE = '{name:40} {datetime:20}'
 FILTER_KEYS = ('encrypted', 'compressed', 'content_type', 'database')
@@ -28,7 +28,7 @@ class Command(BaseDbBackupCommand):
 
     def handle(self, **options):
         self.quiet = options.get('quiet')
-        self.storage = BaseStorage.storage_factory()
+        self.storage = get_storage()
         files_attr = self.get_backup_attrs(options)
         if not self.quiet:
             title = ROW_TEMPLATE.format(name='Name', datetime='Datetime')

dbbackup/management/commands/mediabackup.py

@@ -12,7 +12,8 @@
 
 from ._base import BaseDbBackupCommand
 from ... import utils
-from ...storage.base import BaseStorage, StorageError
+from ...storage import get_storage, StorageError
+from ... import settings
 
 
 class Command(BaseDbBackupCommand):
@@ -43,7 +44,7 @@ def handle(self, *args, **options):
         self.path = options.get('output_path')
         try:
             self.media_storage = get_storage_class()()
-            self.storage = BaseStorage.storage_factory()
+            self.storage = get_storage()
             self.backup_mediafiles()
             if options.get('clean'):
                 self._cleanup_old_backups()

dbbackup/management/commands/mediarestore.py

@@ -8,7 +8,7 @@
 from django.core.files.storage import get_storage_class
 
 from ._base import BaseDbBackupCommand
-from ...storage.base import BaseStorage, StorageError
+from ...storage import get_storage, StorageError
 from ... import utils
 
 
@@ -39,7 +39,7 @@ def handle(self, *args, **options):
         self.replace = options.get('replace')
         self.passphrase = options.get('passphrase')
         self.interactive = options.get('interactive')
-        self.storage = BaseStorage.storage_factory()
+        self.storage = get_storage()
         self.media_storage = get_storage_class()()
         self._restore_backup()
 

dbbackup/settings.py

@@ -42,7 +42,6 @@
 GPG_RECIPIENT = GPG_ALWAYS_TRUST = getattr(settings, 'DBBACKUP_GPG_RECIPIENT', None)
 
 STORAGE = getattr(settings, 'DBBACKUP_STORAGE', 'dbbackup.storage.filesystem_storage')
-BUILTIN_STORAGE = getattr(settings, 'DBBACKUP_BUILTIN_STORAGE', None)
 STORAGE_OPTIONS = getattr(settings, 'DBBACKUP_STORAGE_OPTIONS', {})
 
 CONNECTORS = getattr(settings, 'DBBACKUP_CONNECTORS', {})
@@ -63,25 +62,3 @@
 if hasattr(settings, 'DBBACKUP_FAKE_HOST'):  # pragma: no cover
     warnings.warn("DBBACKUP_FAKE_HOST is deprecated, use DBBACKUP_HOSTNAME", DeprecationWarning)
     HOSTNAME = settings.DBBACKUP_FAKE_HOST
-
-UNSED_AWS_SETTINGS = ('DIRECTORY',)
-DEPRECATED_AWS_SETTINGS = (
-    ('BUCKET', 'bucket_name'),
-    ('ACCESS_KEY', 'access_key'),
-    ('SECRET_KEY', 'secret_key'),
-    ('DOMAIN', 'host'),
-    ('IS_SECURE', 'use_ssl'),
-    ('SERVER_SIDE_ENCRYPTION', 'encryption'),
-)
-if hasattr(settings, 'DBBACKUP_S3_BUCKET'):
-    for old_suffix, new_key in DEPRECATED_AWS_SETTINGS:
-        old_key = 'DBBACKUP_S3_%s' % old_suffix
-        if hasattr(settings, old_key):
-            STORAGE_OPTIONS[new_key] = getattr(settings, old_key)
-            msg = "%s is deprecated, use DBBACKUP_STORAGE_OPTIONS['%s']" % (old_key, new_key)
-            warnings.warn(msg, DeprecationWarning)
-    for old_suffix in UNSED_AWS_SETTINGS:
-        if hasattr(settings, 'DBBACKUP_S3_%s' % old_suffix):
-            msg = "DBBACKUP_S3_%s is now useless" % old_suffix
-            warnings.warn(msg, DeprecationWarning)
-    del old_suffix, new_key

dbbackup/storage/base.py → dbbackup/storage.py

@@ -1,10 +1,10 @@
 """
-Abstract Storage class.
+Backup Storage class.
 """
 import logging
-from importlib import import_module
 from django.core.exceptions import ImproperlyConfigured
-from .. import settings, utils
+from django.core.files.storage import get_storage_class
+from . import settings, utils
 
 
 def get_storage(path=None, options=None):
@@ -23,13 +23,11 @@ def get_storage(path=None, options=None):
     :rtype: :class:`.Storage`
     """
     path = path or settings.STORAGE
-    option = options or {}
     options = options or settings.STORAGE_OPTIONS
     if not path:
         raise ImproperlyConfigured('You must specify a storage class using '
                                    'DBBACKUP_STORAGE settings.')
-    storage_module = import_module(path)
-    return storage_module.Storage(**options)
+    return Storage(path, **options)
 
 
 class StorageError(Exception):
@@ -40,38 +38,51 @@ class FileNotFound(StorageError):
     pass
 
 
-class BaseStorage(object):
+class Storage(object):
     """Abstract storage class."""
-
     @property
     def logger(self):
         if not hasattr(self, '_logger'):
             self._logger = logging.getLogger('dbbackup.storage')
         return self._logger
 
-    def __init__(self, server_name=None):
-        if not self.name:
-            raise Exception("Programming Error: storage.name not defined.")
+    def __init__(self, storage_path=None, **options):
+        """
+        Initialize a Django Storage instance with given options.
+
+        :param storage_path: Path to a Django Storage class with dot style
+                             If ``None``, ``settings.DBBACKUP_BUILTIN_STORAGE``
+                             will be used.
+        :type storage_path: str
+        """
+        self._storage_path = storage_path or settings.STORAGE
+        options = options.copy()
+        options.update(settings.STORAGE_OPTIONS)
+        options = dict([(key.lower(), value) for key, value in options.items()])
+        self.storageCls = get_storage_class(self._storage_path)
+        self.storage = self.storageCls(**options)
+        self.name = self.storageCls.__name__
 
     def __str__(self):
         return self.name
 
-    # TODO: Remove in favor of get_storage
-    @classmethod
-    def storage_factory(cls):
-        return get_storage()
-
-    def backup_dir(self):
-        raise NotImplementedError("Programming Error: backup_dir() not defined.")
-
     def delete_file(self, filepath):
-        raise NotImplementedError("Programming Error: delete_file() not defined.")
+        self.logger.debug('Deleting file %s', filepath)
+        self.storage.delete(name=filepath)
+
+    def list_directory(self):
+        return self.storage.listdir('')[1]
 
     def write_file(self, filehandle, filename):
-        raise NotImplementedError("Programming Error: write_file() not defined.")
+        self.logger.debug('Writing file %s', filename)
+        self.storage.save(name=filename, content=filehandle)
 
     def read_file(self, filepath):
-        raise NotImplementedError("Programming Error: read_file() not defined.")
+        self.logger.debug('Reading file %s', filepath)
+        file_ = self.storage.open(name=filepath, mode='rb')
+        if file_.name is None:
+            file_.name = filepath
+        return file_
 
     def list_backups(self, encrypted=None, compressed=None, content_type=None,
                      database=None):