debug DIRT migration (#301 WIP)

Computational-Plant-Science · May 26, 2022 · a700c17 · a700c17
1 parent d9d155b
commit a700c17
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 11 deletions.
diff --git a/plantit/front_end/src/components/navigation.vue b/plantit/front_end/src/components/navigation.vue
@@ -1011,10 +1011,10 @@ export default {
             'notificationsRead',
             'notificationsUnread',
         ]),
-        downloadedFolders() {
-            if (this.profileLoading || this.profile === null || this.profile.migration.downloads.length === 0) return [];
+        storageFolders() {
+            if (this.profileLoading || this.profile === null || this.profile.migration.storage.length === 0) return [];
             let grouped = this.groupBy(
-                this.profile.migration.downloads,
+                this.profile.migration.storage,
                 'folder'
             );
             return Object.entries(grouped).map((pair) => {

diff --git a/plantit/plantit/celery_tasks.py b/plantit/plantit/celery_tasks.py
@@ -927,8 +927,7 @@ def agents_healthchecks():
 # DIRT migration
 
 
-class DownloadedFile(TypedDict):
-    folder: str
+class StorageDirectory(TypedDict):
     name: str
 
 
@@ -960,6 +959,7 @@ async def push_migration_event(user: User, migration: Migration):
 SELECT_ROOT_COLLECTION_SOIL_K = """SELECT * FROM field_data_field_collection_soil_potassium WHERE entity_id = :entity_id"""
 SELECT_ROOT_COLLECTION_PESTICIDES = """SELECT * FROM field_data_field_collection_pesticides WHERE entity_id = :entity_id"""
 
+
 @app.task(bind=True)
 def migrate_dirt_datasets(self, username: str):
     try:
@@ -1005,7 +1005,7 @@ def migrate_dirt_datasets(self, username: str):
             Path(staging_dir).mkdir(parents=True, exist_ok=True)
 
             # keep track of progress so we can update the UI in real time
-            downloads = []
+            storage = []
             uploads = []
 
             # transfer all the user's datasets to temporary staging directory, 1 file at a time (to preserve local disk space)
@@ -1016,7 +1016,7 @@ def migrate_dirt_datasets(self, username: str):
 
                 # get Drupal managed file IDs for each root image file in the current folder
                 # (files are stored under CAS username if CAS was used for the user's DIRT account, otherwise under their full name)
-                values = [{'path': f"public://{user.username if profile.dirt_name is None else profile.dirt_name}/root-images/{folder}/%"}]
+                values = {'path': f"public://{user.username if profile.dirt_name is None else profile.dirt_name}/root-images/{folder}/%"}
                 rows = async_to_sync(db.fetch_all)(query=SELECT_MANAGED_FILE_BY_PATH, values=values)
                 managed_files = [{
                     'fid': row['fid'],
@@ -1124,6 +1124,10 @@ def migrate_dirt_datasets(self, username: str):
 
                     # TODO attach metadata to each file
 
+                storage.append(StorageDirectory(name=folder_name))
+                migration.storage = json.dumps(storage)
+                async_to_sync(push_migration_event)(user, migration)
+
                 # persist migration status
                 migration.save()
 

diff --git a/plantit/plantit/queries.py b/plantit/plantit/queries.py
@@ -755,7 +755,7 @@ def migration_to_dict(migration: Migration) -> dict:
         'completed': None if migration.completed is None else migration.completed.isoformat(),
         'target_path': migration.target_path,
         'num_folders': migration.num_folders,
-        'downloads': json.loads(migration.downloads),
+        'storage': json.loads(migration.storage),
         'uploads': json.loads(migration.uploads)
     }
 

diff --git a/plantit/plantit/users/models.py b/plantit/plantit/users/models.py
@@ -29,5 +29,5 @@ class Migration(models.Model):
     completed = models.DateTimeField(null=True, blank=True)
     target_path = models.CharField(max_length=255, null=True, blank=True)
     num_folders = models.IntegerField(null=True, blank=True)
-    downloads = models.JSONField(null=True, blank=True)
+    storage = models.JSONField(null=True, blank=True)
     uploads = models.JSONField(null=True, blank=True)
diff --git a/plantit/plantit/users/views.py b/plantit/plantit/users/views.py
@@ -262,7 +262,7 @@ def get_current(self, request):
 
         migration, created = Migration.objects.get_or_create(profile=user.profile)
         if created:
-            migration.downloads = json.dumps([])
+            migration.storage = json.dumps([])
             migration.uploads = json.dumps([])
             migration.save()
 
@@ -434,7 +434,7 @@ def start_dirt_migration(self, request):
         start = timezone.now()
         migration.started = start
         migration.target_path = root_collection_path
-        migration.downloads = json.dumps([])
+        migration.storage = json.dumps([])
         migration.uploads = json.dumps([])
         migration.save()
         profile.save()