diff --git a/requirements.txt b/requirements.txt index 7571163..b83682a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ Django>=4.2,<4.3 apache-libcloud>=3.6,<3.7 django-cors-headers>=3.13,<3.14 django-debug-toolbar>=4.0,<4.1 +django-dynamic-storage==0.1.1 django-environ>=0.9,<1.0 django-grpc>=1.0,<1.1 django-htmx>=1.12,<1.13 diff --git a/say/core/migrations/0001_dsws.py b/say/core/migrations/0001_dsws.py index 133e82d..c436fd8 100644 --- a/say/core/migrations/0001_dsws.py +++ b/say/core/migrations/0001_dsws.py @@ -3,7 +3,7 @@ from django.conf import settings from django.db import migrations, models import django.db.models.deletion -import say.dynamic_storage.models +import dynamic_storage.models import taggit.managers import wagtail.images.models import wagtail.models.collections @@ -60,7 +60,7 @@ class Migration(migrations.Migration): ), ( "file", - say.dynamic_storage.models.DynamicImageField( + dynamic_storage.models.DynamicImageField( height_field="height", upload_to=wagtail.images.models.get_upload_to, verbose_name="file", diff --git a/say/core/migrations/0002_alter_dswrendition_file.py b/say/core/migrations/0002_alter_dswrendition_file.py index 7fa869a..fe32412 100644 --- a/say/core/migrations/0002_alter_dswrendition_file.py +++ b/say/core/migrations/0002_alter_dswrendition_file.py @@ -5,7 +5,7 @@ from django.db import migrations import wagtail.images.models -import say.dynamic_storage.models +import dynamic_storage.models def delete_all_renditions(apps, schema_editor): @@ -28,7 +28,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name="dswrendition", name="file", - field=say.dynamic_storage.models.DynamicImageField( + field=dynamic_storage.models.DynamicImageField( height_field="height", upload_to=wagtail.images.models.get_rendition_upload_to, width_field="width", diff --git a/say/core/migrations/0003_dswdocument.py b/say/core/migrations/0003_dswdocument.py index c303d6b..cf78077 100644 --- a/say/core/migrations/0003_dswdocument.py +++ b/say/core/migrations/0003_dswdocument.py @@ -3,7 +3,7 @@ from django.conf import settings from django.db import migrations, models import django.db.models.deletion -import say.dynamic_storage.models +import dynamic_storage.models import taggit.managers import wagtail.models.collections import wagtail.search.index @@ -42,7 +42,7 @@ class Migration(migrations.Migration): ), ( "file", - say.dynamic_storage.models.DynamicFileField( + dynamic_storage.models.DynamicFileField( upload_to="documents", verbose_name="file" ), ), diff --git a/say/core/models.py b/say/core/models.py index 84a7bdc..43ab288 100644 --- a/say/core/models.py +++ b/say/core/models.py @@ -12,13 +12,12 @@ get_rendition_upload_to, ) -from grapple.models import GraphQLImage, GraphQLInt, GraphQLString - -from say.dynamic_storage.models import ( +from dynamic_storage.models import ( DynamicFileField, DynamicImageField, DynamicImageFieldFile, ) +from grapple.models import GraphQLImage, GraphQLInt, GraphQLString class Monkey: diff --git a/say/core/signals.py b/say/core/signals.py index bc31348..b0dab73 100644 --- a/say/core/signals.py +++ b/say/core/signals.py @@ -1,8 +1,8 @@ from django.dispatch import receiver -from say.dynamic_storage.models import DynamicFieldFile -from say.dynamic_storage.signals import pre_dynamic_file_save -from say.dynamic_storage.storage import DynamicStorage +from dynamic_storage.models import DynamicFieldFile +from dynamic_storage.signals import pre_dynamic_file_save +from dynamic_storage.storage import DynamicStorage from . import models diff --git a/say/core/storage.py b/say/core/storage.py index fc7cd09..a6a4b96 100644 --- a/say/core/storage.py +++ b/say/core/storage.py @@ -1,12 +1,12 @@ from django.utils.deconstruct import deconstructible +from dynamic_storage.storage import DynamicStorageMixin from storages.backends.apache_libcloud import LibCloudFile as BaseLibCloudFile from say.customized.libcloud.storage.drivers.minio import MinIOStorageDriver from say.customized.storages.backends.apache_libcloud import ( LibCloudStorage as BaseLibCloudStorage, ) -from say.dynamic_storage.storage import DynamicStorageMixin @deconstructible diff --git a/say/dynamic_storage/__init__.py b/say/dynamic_storage/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/say/dynamic_storage/models.py b/say/dynamic_storage/models.py deleted file mode 100644 index d06199b..0000000 --- a/say/dynamic_storage/models.py +++ /dev/null @@ -1,142 +0,0 @@ -from typing import Any, Dict - -from django.db import models -from django.db.models.fields.files import ( - FieldFile, - FileDescriptor, - ImageField, - ImageFieldFile, - ImageFileDescriptor, -) - -from .signals import pre_dynamic_file_save -from .storage import DynamicStorage, prob - - -# {"name": str, "storage": prob} -jsonfield = Dict[str, Any] - - -class DynamicFieldFile(FieldFile): - def __init__(self, instance, field, name, storage: DynamicStorage = None): - super(DynamicFieldFile, self).__init__(instance, field, name) - self._current_storage = ( - storage # keep track of the storage the file is actually at - ) - self.storage = storage or self.storage - - # To be able to save the file to the appropriate storage - # if providing the save() method with storage argument is not possible - self.destination_storage = None - - def dictionary(self) -> jsonfield: - """Value to be stored in JSONField""" - return {"name": str(self), "storage": self.storage.uninit()} - - def save(self, name, content, /, storage: DynamicStorage = None, save=True): - """ - This is the dynamic storage save - """ - self.destination_storage = storage or self.destination_storage - # Ensure we are not trying to move obj between storages (yet) - assert ( - self._current_storage == self.destination_storage - if self._current_storage - else True - ) - - pre_dynamic_file_save.send( - sender=self.instance.__class__, - instance=self.instance, - field_file=self, - to_storage=self.destination_storage, - ) - - storage = self.destination_storage or self._current_storage or self.storage - - name = self.field.generate_filename(self.instance, name) - self.name = storage.save(name, content, max_length=self.field.max_length) - self.storage = self._current_storage = self.destination_storage = storage - - # This is the replacement for 'setattr(self.instance, self.field.attname, self.name)' - # because instance is not wrote to db yet, reinstantiating will break setting the right storage - getattr(self.instance, self.field.attname).name = self.name - - self._committed = True - - # Save the object because it has changed, unless save is False - if save: - self.instance.save() - - def __getstate__(self): - # Don't know how to get the state related to storage - raise NotImplementedError - - def __setstate__(self, state): - # __getstate__ is not implemented - raise NotImplementedError - - -class DynamicFileDescriptor(FileDescriptor): - def __get__(self, instance, cls=None): - file = super(DynamicFileDescriptor, self).__get__(instance, cls=cls) - # If this value is a dictionary (instance.file = {"name": "path/to/file", **kw}) or {} - # then we simply wrap it with the appropriate attribute class according - # to the file field. [This is DynamicFieldFile for DynamicFileField and - # ImageFieldFile for ImageFields; it's also conceivable that user - # subclasses might also want to subclass the attribute class]. This - # object understands how to convert a jsonfield to a file, and also how to - # handle dict(). - if isinstance(file, dict): - storage_prob: prob = file.get("storage") - storage = ( - DynamicStorage.init(storage_prob) if storage_prob else storage_prob - ) - attr = self.field.attr_class( - instance, self.field, file.get("name"), storage - ) - instance.__dict__[self.field.attname] = attr - - return instance.__dict__[self.field.attname] - - -class DynamicFileField(models.JSONField, models.FileField): - """FileField with json db representation that contain info for dynamic behavior""" - - attr_class = DynamicFieldFile - - descriptor_class = DynamicFileDescriptor - - def get_db_prep_value(self, value, connection, prepared=False): - if value is None: - return value - if not isinstance(value, dict): - value = value.dictionary() - return super(DynamicFileField, self).get_db_prep_value( - value, connection, prepared - ) - - def formfield(self, **kwargs): - return models.FileField.formfield(self, **kwargs) - - def validate(self, value, model_instance): - super(DynamicFileField, self).validate(value.dictionary(), model_instance) - - -class DynamicImageFieldFile(ImageFieldFile, DynamicFieldFile): - def save(self, *args, **kwargs): - super(DynamicImageFieldFile, self).save(*args, **kwargs) - # Since reinstantiating the FieldFile after calling save() is not the case anymore - # (due to delete 'setattr(self.instance, self.field.attname, self.name)') - self.field.update_dimension_fields(self.instance, force=True) - - -class DynamicImageFileDescriptor(ImageFileDescriptor, DynamicFileDescriptor): - pass - - -class DynamicImageField(ImageField, DynamicFileField): - """ImageField with json db representation that contain info for dynamic behavior""" - - attr_class = DynamicImageFieldFile - descriptor_class = DynamicImageFileDescriptor diff --git a/say/dynamic_storage/signals.py b/say/dynamic_storage/signals.py deleted file mode 100644 index f8ed3e0..0000000 --- a/say/dynamic_storage/signals.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.dispatch import Signal - - -# provides args: instance, field_file, to_storage -pre_dynamic_file_save = Signal() diff --git a/say/dynamic_storage/storage.py b/say/dynamic_storage/storage.py deleted file mode 100644 index 60ca5f0..0000000 --- a/say/dynamic_storage/storage.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -import importlib -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict - -from django.core.files.storage import Storage - - -# {"import_path": str, "constructor": dict} -prob = Dict[str, Any] - - -class DynamicStorageMixin(ABC): - @abstractmethod - def init_params(self) -> dict: - """parameters for calling __init__ on storage class""" - ... - - def uninit(self) -> prob: - """get the required properties for future initialization""" - return { - "import_path": f"{self.__class__.__module__}.{self.__class__.__qualname__}", - "constructor": self.init_params(), - } - - @classmethod - def init(cls, probs: prob) -> DynamicStorageMixin: - """initialize storage""" - module_name = ".".join(probs["import_path"].split(".")[:-1]) - class_name = probs["import_path"].split(".")[-1] - StorageClass: Callable = getattr( - importlib.import_module(module_name), class_name - ) - return StorageClass(**probs["constructor"]) - - -class DynamicStorage(DynamicStorageMixin, Storage): - ... diff --git a/say/storage/storage.py b/say/storage/storage.py index 9839615..75b5c05 100644 --- a/say/storage/storage.py +++ b/say/storage/storage.py @@ -7,10 +7,10 @@ from django.core.files.storage import Storage from django.utils.deconstruct import deconstructible +from dynamic_storage.storage import DynamicStorageMixin from pydantic import BaseModel, ValidationError, conint, constr from say.core.storage import MinioStorage -from say.dynamic_storage.storage import DynamicStorageMixin class StorageDoesNotExists(Exception):