separate specification for DIRT migration env vars (#301)

Computational-Plant-Science · May 16, 2022 · 7fc7b39 · 7fc7b39
1 parent 9a1da54
commit 7fc7b39
Show file tree

Hide file tree

Showing 4 changed files with 48 additions and 24 deletions.
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -63,8 +63,11 @@ services:
       - AGENTS_HEALTHCHECKS_MINUTES=${AGENTS_HEALTHCHECKS_MINUTES}
       - AGENTS_HEALTHCHECKS_SAVED=${AGENTS_HEALTHCHECKS_SAVED}
       - HTTP_TIMEOUT=${HTTP_TIMEOUT}
-      - DIRT_STAGING_DIR=${DIRT_STAGING_DIR}
-      - DIRT_DATA_DIR=${DIRT_DATA_DIR}
+      - DIRT_MIGRATION_STAGING_DIR=${DIRT_MIGRATION_STAGING_DIR}
+      - DIRT_MIGRATION_DATA_DIR=${DIRT_MIGRATION_DATA_DIR}
+      - DIRT_MIGRATION_HOST=${DIRT_MIGRATION_HOST}
+      - DIRT_MIGRATION_PORT=${DIRT_MIGRATION_PORT}
+      - DIRT_MIGRATION_USERNAME=${DIRT_MIGRATION_USERNAME}
     command: ["/code/scripts/wait-for-postgres.sh", "postgres", "python", "manage.py", "runserver", "0.0.0.0:3000"]
     depends_on:
       - postgres
@@ -167,8 +170,11 @@ services:
       - AGENTS_HEALTHCHECKS_MINUTES=${AGENTS_HEALTHCHECKS_MINUTES}
       - AGENTS_HEALTHCHECKS_SAVED=${AGENTS_HEALTHCHECKS_SAVED}
       - HTTP_TIMEOUT=${HTTP_TIMEOUT}
-      - DIRT_STAGING_DIR=${DIRT_STAGING_DIR}
-      - DIRT_DATA_DIR=${DIRT_DATA_DIR}
+      - DIRT_MIGRATION_STAGING_DIR=${DIRT_MIGRATION_STAGING_DIR}
+      - DIRT_MIGRATION_DATA_DIR=${DIRT_MIGRATION_DATA_DIR}
+      - DIRT_MIGRATION_HOST=${DIRT_MIGRATION_HOST}
+      - DIRT_MIGRATION_PORT=${DIRT_MIGRATION_PORT}
+      - DIRT_MIGRATION_USERNAME=${DIRT_MIGRATION_USERNAME}
     healthcheck:
       test: celery -A plantit status | grep "OK"
       interval: 60s
@@ -240,8 +246,11 @@ services:
       - AGENTS_HEALTHCHECKS_MINUTES=${AGENTS_HEALTHCHECKS_MINUTES}
       - AGENTS_HEALTHCHECKS_SAVED=${AGENTS_HEALTHCHECKS_SAVED}
       - HTTP_TIMEOUT=${HTTP_TIMEOUT}
-      - DIRT_STAGING_DIR=${DIRT_STAGING_DIR}
-      - DIRT_DATA_DIR=${DIRT_DATA_DIR}
+      - DIRT_MIGRATION_STAGING_DIR=${DIRT_MIGRATION_STAGING_DIR}
+      - DIRT_MIGRATION_DATA_DIR=${DIRT_MIGRATION_DATA_DIR}
+      - DIRT_MIGRATION_HOST=${DIRT_MIGRATION_HOST}
+      - DIRT_MIGRATION_PORT=${DIRT_MIGRATION_PORT}
+      - DIRT_MIGRATION_USERNAME=${DIRT_MIGRATION_USERNAME}
     healthcheck:
       test: celery -A plantit status | grep "OK"
       interval: 60s

diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
@@ -67,8 +67,11 @@ services:
       - AGENTS_HEALTHCHECKS_MINUTES=${AGENTS_HEALTHCHECKS_MINUTES}
       - AGENTS_HEALTHCHECKS_SAVED=${AGENTS_HEALTHCHECKS_SAVED}
       - HTTP_TIMEOUT=${HTTP_TIMEOUT}
-      - DIRT_STAGING_DIR=${DIRT_STAGING_DIR}
-      - DIRT_DATA_DIR=${DIRT_DATA_DIR}
+      - DIRT_MIGRATION_STAGING_DIR=${DIRT_MIGRATION_STAGING_DIR}
+      - DIRT_MIGRATION_DATA_DIR=${DIRT_MIGRATION_DATA_DIR}
+      - DIRT_MIGRATION_HOST=${DIRT_MIGRATION_HOST}
+      - DIRT_MIGRATION_PORT=${DIRT_MIGRATION_PORT}
+      - DIRT_MIGRATION_USERNAME=${DIRT_MIGRATION_USERNAME}
     command: ["daphne", "-b", "0.0.0.0", "-p", "8000", "plantit.asgi:application"]
     networks:
       - ngnix_network
@@ -145,8 +148,11 @@ services:
       - AGENTS_HEALTHCHECKS_MINUTES=${AGENTS_HEALTHCHECKS_MINUTES}
       - AGENTS_HEALTHCHECKS_SAVED=${AGENTS_HEALTHCHECKS_SAVED}
       - HTTP_TIMEOUT=${HTTP_TIMEOUT}
-      - DIRT_STAGING_DIR=${DIRT_STAGING_DIR}
-      - DIRT_DATA_DIR=${DIRT_DATA_DIR}
+      - DIRT_MIGRATION_STAGING_DIR=${DIRT_MIGRATION_STAGING_DIR}
+      - DIRT_MIGRATION_DATA_DIR=${DIRT_MIGRATION_DATA_DIR}
+      - DIRT_MIGRATION_HOST=${DIRT_MIGRATION_HOST}
+      - DIRT_MIGRATION_PORT=${DIRT_MIGRATION_PORT}
+      - DIRT_MIGRATION_USERNAME=${DIRT_MIGRATION_USERNAME}
     healthcheck:
       test: celery -A plantit status | grep "OK"
       interval: 60s
@@ -219,8 +225,11 @@ services:
       - AGENTS_HEALTHCHECKS_MINUTES=${AGENTS_HEALTHCHECKS_MINUTES}
       - AGENTS_HEALTHCHECKS_SAVED=${AGENTS_HEALTHCHECKS_SAVED}
       - HTTP_TIMEOUT=${HTTP_TIMEOUT}
-      - DIRT_STAGING_DIR=${DIRT_STAGING_DIR}
-      - DIRT_DATA_DIR=${DIRT_DATA_DIR}
+      - DIRT_MIGRATION_STAGING_DIR=${DIRT_MIGRATION_STAGING_DIR}
+      - DIRT_MIGRATION_DATA_DIR=${DIRT_MIGRATION_DATA_DIR}
+      - DIRT_MIGRATION_HOST=${DIRT_MIGRATION_HOST}
+      - DIRT_MIGRATION_PORT=${DIRT_MIGRATION_PORT}
+      - DIRT_MIGRATION_USERNAME=${DIRT_MIGRATION_USERNAME}
     healthcheck:
       test: celery -A plantit status | grep "OK"
       interval: 60s

diff --git a/plantit/plantit/celery_tasks.py b/plantit/plantit/celery_tasks.py
@@ -967,15 +967,15 @@ def migrate_dirt_datasets(self, username: str):
 
     # create SSH/SFTP client and open SFTP connection
     ssh = SSH(
-        host='tucco.cyverse.org',
-        port=1657,
-        username=settings.CYVERSE_USERNAME,
-        pkey=str(get_user_private_key_path(settings.CYVERSE_USERNAME)))
+        host=settings.DIRT_MIGRATION_HOST,
+        port=settings.DIRT_MIGRATION_PORT,
+        username=settings.DIRT_MIGRATION_USERNAME,
+        pkey=str(get_user_private_key_path(settings.DIRT_MIGRATION_USERNAME)))
     with ssh:
         with ssh.client.open_sftp() as sftp:
 
             # list the user's datasets on the DIRT server
-            user_dir = join(settings.DIRT_DATA_DIR, username, 'root-images')
+            user_dir = join(settings.DIRT_MIGRATION_DATA_DIR, username, 'root-images')
             datasets = [folder for folder in sftp.listdir(user_dir)]
             logger.info(f"User {username} has {len(datasets)} DIRT folders: {', '.join(datasets)}")
 
@@ -998,13 +998,13 @@ def migrate_dirt_datasets(self, username: str):
                 logger.info(f"User {username} folder {folder} has {len(files)} files: {', '.join(files)}")
 
                 # create temp local folder for this dataset
-                staging_dir = join(settings.DIRT_STAGING_DIR, folder)
+                staging_dir = join(settings.DIRT_MIGRATION_STAGING_DIR, folder)
                 Path(staging_dir).mkdir(parents=True, exist_ok=True)
 
                 # download files
                 for file in files:
                     file_path = join(folder_name, file)
-                    sftp.get(file_path, join(settings.DIRT_STAGING_DIR, folder, file))
+                    sftp.get(file_path, join(settings.DIRT_MIGRATION_STAGING_DIR, folder, file))
 
                     # push download status update to UI
                     downloads.append(DownloadedFile(name=file, folder=folder))
@@ -1026,7 +1026,7 @@ def migrate_dirt_datasets(self, username: str):
 
                 # upload all files to collection
                 client.upload_directory(
-                    from_path=join(settings.DIRT_STAGING_DIR, folder),
+                    from_path=join(settings.DIRT_MIGRATION_STAGING_DIR, folder),
                     to_prefix=collection_path)
 
                 # get ID of newly created collection

diff --git a/plantit/plantit/settings.py b/plantit/plantit/settings.py
@@ -48,8 +48,11 @@
 assert 'STATS_WINDOW_WIDTH_DAYS' in os.environ, f"{missing_variable} STATS_WINDOW_WIDTH_DAYS"
 assert 'DOCKER_USERNAME' in os.environ, f"{missing_variable} DOCKER_USERNAME"
 assert 'DOCKER_PASSWORD' in os.environ, f"{missing_variable} DOCKER_PASSWORD"
-assert 'DIRT_DATA_DIR' in os.environ, f"{missing_variable} DIRT_DATA_DIR"
-assert 'DIRT_STAGING_DIR' in os.environ, f"{missing_variable} DIRT_STAGING_DIR"
+assert 'DIRT_MIGRATION_DATA_DIR' in os.environ, f"{missing_variable} DIRT_MIGRATION_DATA_DIR"
+assert 'DIRT_MIGRATION_STAGING_DIR' in os.environ, f"{missing_variable} DIRT_MIGRATION_STAGING_DIR"
+assert 'DIRT_MIGRATION_HOST' in os.environ, f"{missing_variable} DIRT_MIGRATION_HOST"
+assert 'DIRT_MIGRATION_PORT' in os.environ, f"{missing_variable} DIRT_MIGRATION_PORT"
+assert 'DIRT_MIGRATION_USERNAME' in os.environ, f"{missing_variable} DIRT_MIGRATION_USERNAME"
 
 # global Celery task timeout
 CELERYD_TASK_SOFT_TIME_LIMIT = 60
@@ -92,8 +95,11 @@
 STATS_WINDOW_WIDTH_DAYS = os.environ.get("STATS_WINDOW_WIDTH_DAYS")
 DOCKER_USERNAME = os.environ.get("DOCKER_USERNAME")
 DOCKER_PASSWORD = os.environ.get("DOCKER_PASSWORD")
-DIRT_STAGING_DIR = os.environ.get("DIRT_STAGING_DIR")
-DIRT_DATA_DIR = os.environ.get("DIRT_DATA_DIR")
+DIRT_MIGRATION_STAGING_DIR = os.environ.get("DIRT_MIGRATION_STAGING_DIR")
+DIRT_MIGRATION_DATA_DIR = os.environ.get("DIRT_MIGRATION_DATA_DIR")
+DIRT_MIGRATION_HOST = os.environ.get("DIRT_MIGRATION_HOST")
+DIRT_MIGRATION_PORT = os.environ.get("DIRT_MIGRATION_PORT")
+DIRT_MIGRATION_USERNAME = os.environ.get("DIRT_MIGRATION_USERNAME")
 
 if not DEBUG:
     SECURE_SSL_REDIRECT = os.environ.get('DJANGO_SECURE_SSL_REDIRECT')