Perf/clean medias

backend/geonature/core/gn_commons/medias/routes.py

@@ -90,8 +90,6 @@ def insert_or_update_media(id_media=None):
 
     m = TMediaRepository(data=data, file=file, id_media=id_media).create_or_update_media()
 
-    TMediumRepository.sync_medias()
-
     return m.as_dict()
 
 
@@ -106,8 +104,6 @@ def delete_media(id_media):
 
     TMediaRepository(id_media=id_media).delete()
 
-    TMediumRepository.sync_medias()
-
     return {"resp": "media {} deleted".format(id_media)}
 
 

backend/geonature/core/gn_commons/routes.py

@@ -26,6 +26,7 @@
 from geonature.core.gn_permissions import decorators as permissions
 from geonature.core.gn_permissions.decorators import login_required
 from geonature.core.gn_permissions.tools import get_scopes_by_action
+import geonature.core.gn_commons.tasks  # noqa: F401
 
 from shapely.geometry import asShape
 from geoalchemy2.shape import from_shape

backend/geonature/core/gn_commons/tasks.py

@@ -0,0 +1,33 @@
+from celery.schedules import crontab
+from celery.utils.log import get_task_logger
+
+from geonature.core.gn_commons.repositories import TMediumRepository
+from geonature.utils.celery import celery_app
+from geonature.utils.config import config
+
+logger = get_task_logger(__name__)
+
+
+@celery_app.on_after_finalize.connect
+def setup_periodic_tasks(sender, **kwargs):
+    ct = config["MEDIA_CLEAN_CRONTAB"]
+    if ct:
+        minute, hour, day_of_month, month_of_year, day_of_week = ct.split(" ")
+        sender.add_periodic_task(
+            crontab(
+                minute=minute,
+                hour=hour,
+                day_of_week=day_of_week,
+                day_of_month=day_of_month,
+                month_of_year=month_of_year,
+            ),
+            clean_attachments.s(),
+            name="clean medias",
+        )
+
+
+@celery_app.task(bind=True)
+def clean_attachments(self):
+    logger.info("Cleaning medias...")
+    TMediumRepository.sync_medias()
+    logger.info("Medias cleaned")

backend/geonature/tests/fixtures.py

@@ -50,6 +50,7 @@
     "medium",
     "module",
     "perm_object",
+    "celery_eager",
 ]
 
 
@@ -371,8 +372,13 @@ def create_media(media_path=""):
 
 @pytest.fixture
 def medium(app):
+    # FIXME: find a better way to get the id_media that will be created
+    new_id_media = (db.session.query(func.max(TMedias.id_media)).scalar() or 0) + 1
     image = Image.new("RGBA", size=(1, 1), color=(155, 0, 0))
-    with tempfile.NamedTemporaryFile(dir=TMedias.base_dir(), suffix=".png") as f:
+    # Delete = false since it will be done automatically
+    with tempfile.NamedTemporaryFile(
+        dir=TMedias.base_dir(), prefix=f"{new_id_media}_", suffix=".png", delete=False
+    ) as f:
         image.save(f, "png")
         yield create_media(media_path=str(f.name))
 

backend/geonature/tests/test_gn_commons.py

@@ -14,7 +14,7 @@
 from geonature.core.gn_commons.models import TAdditionalFields, TMedias, TPlaces, BibTablesLocation
 from geonature.core.gn_commons.models.base import TModules, TParameters, BibWidgets
 from geonature.core.gn_commons.repositories import TMediaRepository
-
+from geonature.core.gn_commons.tasks import clean_attachments
 from geonature.core.gn_permissions.models import TObjects
 from geonature.utils.env import db
 from geonature.utils.errors import GeoNatureError
@@ -539,3 +539,23 @@ def test_api_get_id_table_location_not_found(self):
 
         assert response.status_code == 204  # No content
         assert response.json is None
+
+
+@pytest.mark.usefixtures("temporary_transaction")
+class TestTasks:
+    def test_clean_attachements(self, monkeypatch, celery_eager, medium):
+        # Monkey patch the __before_commit_delete not to remove file
+        # when deleting the medium, so the clean_attachments can work
+        def mock_delete_media(self):
+            return None
+
+        monkeypatch.setattr(TMedias, "__before_commit_delete__", mock_delete_media)
+
+        # Remove media to trigger the cleaning
+        db.session.delete(medium)
+        db.session.commit()
+
+        clean_attachments()
+
+        # File should be removed
+        assert not Path(medium.media_path).is_file()

backend/geonature/utils/config_schema.py

@@ -555,6 +555,7 @@ class GnGeneralSchemaConf(Schema):
     HOME = fields.Nested(HomeConfig, load_default=HomeConfig().load({}))
     NOTIFICATIONS_ENABLED = fields.Boolean(load_default=True)
     PROFILES_REFRESH_CRONTAB = fields.String(load_default="0 3 * * *")
+    MEDIA_CLEAN_CRONTAB = fields.String(load_default="0 1 * * *")
 
     @validates_schema
     def validate_enable_sign_up(self, data, **kwargs):

config/default_config.toml.example

@@ -73,6 +73,8 @@ MAIL_ON_ERROR = false
 # Notifications (true / false)
 NOTIFICATIONS_ENABLED = true
 
+MEDIA_CLEAN_CRONTAB = "0 1 * * *"
+
 [USERSHUB]
     # URL de l'application Usershub
     URL_USERSHUB = "http://127.0.0.1:5001"