Skip to content

Commit

Permalink
DIRT migration complete? (#301)
Browse files Browse the repository at this point in the history
  • Loading branch information
@wpbonelli
wpbonelli committed May 15, 2022
1 parent a325de0 commit 9a1da54
Show file tree
Hide file tree
Showing 8 changed files with 192 additions and 103 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ plantit/settings.py
config/ssh/*
config/certbot/*
agent_keys
annotate

#Documentation
!docs/build/
Expand Down
97 changes: 87 additions & 10 deletions plantit/front_end/src/components/navigation.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -759,9 +759,77 @@
:header-border-variant="profile.darkMode ? 'dark' : 'white'"
:footer-border-variant="profile.darkMode ? 'dark' : 'white'"
:body-text-variant="profile.darkMode ? 'white' : 'dark'"
:ok-disabled="true"
hide-footer
busy
>
<p>{{migrationData}}</p>
<b-row v-if="profile.dirtMigrationCompleted === null">
<b-col
>
<p>You haven't migrated your datasets from DIRT yet.</p>
<b-button
:disabled="
migrationData !== null && migrationData.started !== null
"
@click="
startDirtMigration
"
:variant="
profile.darkMode
? 'outline-success'
: 'success'
"
block
>
<b-spinner
small
v-if="
migrationData.started !== null
"
label="Running..."
variant="dark"
class="
mr-2
"
></b-spinner
><i
v-else
class="
fas
fa-chevron-right
fa-fw
mr-1
"
></i>
Start Migration</b-button
>
<p v-if="migrationData.started !== null">
<br/>
<b>Started:</b> {{ prettify(migrationData.started) }}
<br/>
<b>Downloaded files:</b> {{ migrationData.downloads.length }}
<b-list-group>
<b-list-group-item :variant="profile.darkMode ? 'dark' : 'light'" v-for="file in migrationData.downloads" v-bind:key="`${file.folder} ${file.name}`">
<i class="fas text-success fa-check-double fa-1x fa-fw"></i> {{ file.folder }}/{{file.name}}
</b-list-group-item>
</b-list-group>
<b>Uploaded datasets:</b> {{ migrationData.uploads.length }}/{{ migrationData.num_folders }}
<br/>
<b-progress :value="migrationData.uploads.length" :max="migrationData.num_folders" show-progress animated variant="success"></b-progress>
</p>
</b-col
>
</b-row>
<b-row v-else><b-col>
<p>
Your datasets have been successfully migrated from DIRT.
</p>
<p>
<b>Started:</b> {{ prettify(profile.dirtMigrationStarted) }}
<br/>
<b>Completed:</b> {{ prettify(profile.dirtMigrationCompleted) }}
</p>
</b-col>
</b-row>
</b-modal>
<b-modal
id="feedback"
Expand Down Expand Up @@ -873,7 +941,14 @@ export default {
dismissCountDown: 0,
maintenanceWindows: [],
// DIRT migration
migrationData: null
migrationData: {
started: null,
completed: null,
target_path: null,
num_folders: null,
downloads: [],
uploads: []
}
};
},
computed: {
Expand Down Expand Up @@ -964,6 +1039,9 @@ export default {
alerts() {
this.dismissCountDown = this.dismissSecs;
},
migrationData() {
// noop
}
},
methods: {
showDirtMigrationModal() {
Expand All @@ -976,11 +1054,12 @@ export default {
axios
.get(`/apis/v1/users/start_dirt_migration/`)
.then(async (response) => {
this.migrationData = response.data.migration;
await Promise.all([
this.$store.dispatch('user/setDirtMigrationStarted', true),
this.$store.dispatch('user/setDirtMigrationStarted', this.migrationData.started),
this.$store.dispatch('alerts/add', {
variant: 'success',
message: `Started DIRT migration (target collection: ${response.data.target_path})`,
message: `Started DIRT migration (target collection: ${response.data.migration.target_path})`,
guid: guid().toString(),
})
]);
Expand Down Expand Up @@ -1307,15 +1386,13 @@ export default {
}
},
async handleMigrationEvent(migration) {
let data = JSON.parse(migration.data);
this.migrationData = data;
this.migrationData = migration;
// check if completed and update user profile & create an alert if so
let completed = data.completed;
let completed = migration.completed;
if (completed !== null && completed !== undefined) {
await this.$store.dispatch('user/setDirtMigrationCompleted', true);
await this.$store.dispatch('user/setDirtMigrationCompleted', completed);
}
},
async handleNotificationEvent(notification) {
await this.$store.dispatch('notifications/update', notification);
Expand Down
6 changes: 4 additions & 2 deletions plantit/front_end/src/store/user.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,10 @@ export const user = {
state.profileLoading = loading;
},
setDirtMigrationStarted(state, started) {
state.dirtMigrationStarted = started;
state.profile.dirtMigrationStarted = started;
},
setDirtMigrationCompleted(state, completed) {
state.dirtMigrationCompleted = completed;
state.profile.dirtMigrationCompleted = completed;
}
},
actions: {
Expand Down Expand Up @@ -93,6 +93,8 @@ export const user = {
commit('setHints', response.data.django_profile.hints);
commit('setPushNotifications', response.data.django_profile.push_notifications);
commit('setStats', response.data.stats);
commit('setDirtMigrationStarted', response.data.django_profile.dirt_migration_started);
commit('setDirtMigrationCompleted', response.data.django_profile.dirt_migration_completed);
commit('setProfileLoading', false);
})
.catch(error => {
Expand Down
180 changes: 93 additions & 87 deletions plantit/plantit/celery_tasks.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import json
import os
import traceback
from pathlib import Path
from typing import List, TypedDict, Optional
from os import environ
from os.path import join
Expand Down Expand Up @@ -925,7 +927,8 @@ def agents_healthchecks():


class DownloadedFile(TypedDict):
path: str
folder: str
name: str


class UploadedFolder(TypedDict):
Expand Down Expand Up @@ -968,113 +971,116 @@ def migrate_dirt_datasets(self, username: str):
port=1657,
username=settings.CYVERSE_USERNAME,
pkey=str(get_user_private_key_path(settings.CYVERSE_USERNAME)))
with ssh.client.open_sftp() as sftp:

# list the user's datasets on the DIRT server
user_dir = join(settings.DIRT_DATA_DIR, username, 'root-images')
datasets = [folder for folder in sftp.listdir(user_dir)]
new_line = '\n'
logger.info(f"User {username} has {len(datasets)} datasets:{new_line}{new_line.join(datasets)}")

return

# create a client for the CyVerse APIs and create a collection for the migrated DIRT data
client = TerrainClient(access_token=profile.cyverse_access_token)
root_collection_path = f"/iplant/home/{user.username}/dirt_migration"
if client.dir_exists(root_collection_path):
logger.warning(f"Collection {root_collection_path} already exists, aborting DIRT migration for {user.username}")
return
else: client.mkdir(root_collection_path)
with ssh:
with ssh.client.open_sftp() as sftp:

# list the user's datasets on the DIRT server
user_dir = join(settings.DIRT_DATA_DIR, username, 'root-images')
datasets = [folder for folder in sftp.listdir(user_dir)]
logger.info(f"User {username} has {len(datasets)} DIRT folders: {', '.join(datasets)}")

# create a client for the CyVerse APIs and create a collection for the migrated DIRT data
client = TerrainClient(access_token=profile.cyverse_access_token)
root_collection_path = f"/iplant/home/{user.username}/dirt_migration"
if client.dir_exists(root_collection_path):
logger.warning(f"Collection {root_collection_path} already exists, aborting DIRT migration for {user.username}")
return
else: client.mkdir(root_collection_path)

# keep track of progress so we can update the UI in real time
downloads = []
uploads = []

# transfer all the user's datasets to temporary staging directory
for folder in datasets:
folder_name = join(user_dir, folder)
files = [f for f in sftp.listdir(folder_name)]
logger.info(f"User {username} folder {folder} has {len(files)} files: {', '.join(files)}")

# create temp local folder for this dataset
staging_dir = join(settings.DIRT_STAGING_DIR, folder)
Path(staging_dir).mkdir(parents=True, exist_ok=True)

# download files
for file in files:
file_path = join(folder_name, file)
sftp.get(file_path, join(settings.DIRT_STAGING_DIR, folder, file))

# push download status update to UI
downloads.append(DownloadedFile(name=file, folder=folder))
async_to_sync(push_migration_event)(user, Migration(
started=start.isoformat(),
completed=None,
num_folders=len(datasets),
target_path=root_collection_path,
downloads=downloads,
uploads=[]))

# create subcollection for this folder
collection_path = join(root_collection_path, folder.rpartition('/')[2])
if client.dir_exists(collection_path):
logger.warning(f"Collection {collection_path} already exists, aborting DIRT migration for {user.username}")
return
else:
client.mkdir(collection_path)

# keep track of progress so we can update the UI in real time
downloads = []
uploads = []
# upload all files to collection
client.upload_directory(
from_path=join(settings.DIRT_STAGING_DIR, folder),
to_prefix=collection_path)

# transfer all the user's datasets to temporary staging directory
for folder in datasets:
folder_name = join(user_dir, folder)
files = [f for f in sftp.listdir(folder_name)]
logger.info(f"User {username} folder {folder} has {len(files)} datasets:{new_line}{new_line.join(files)}")
# get ID of newly created collection
stat = client.stat(collection_path)
id = stat['id']

# download files
for file in files:
sftp.get(file.filename, join(settings.DIRT_STAGING_DIR, folder, file.filename))
# mark collection as originating from DIRT
client.set_metadata(id, [
f"dirt_migration_timestamp={timezone.now().isoformat()}",
# TODO: anything else we need to add here?
], [])

# push download status update to UI
downloads.append(DownloadedFile(path=file.filename))
# push upload status update to UI
uploads.append(UploadedFolder(path=collection_path, id=id))
async_to_sync(push_migration_event)(user, Migration(
started=start.isoformat(),
completed=None,
num_folders=len(datasets),
target_path=root_collection_path,
downloads=downloads,
uploads=[]))

# create subcollection for this folder
collection_path = join(root_collection_path, folder.rpartition('/')[2])
if client.dir_exists(collection_path):
logger.warning(f"Collection {collection_path} already exists, aborting DIRT migration for {user.username}")
return
else:
client.mkdir(collection_path)

# upload all files to collection
client.upload_directory(
from_path=join(settings.DIRT_STAGING_DIR, folder),
to_prefix=collection_path)
uploads=uploads))

# get ID of newly created collection
stat = client.stat(collection_path)
id = stat['id']
root_collection_id = client.stat(root_collection_path)['id']

# mark collection as originating from DIRT
client.set_metadata(id, [
f"dirt_migration_timestamp={timezone.now().isoformat()}",
# add collection timestamp as metadata
end = timezone.now()
client.set_metadata(root_collection_id, [
f"dirt_migration_timestamp={end.isoformat()}",
# TODO: anything else we need to add here?
])

# push upload status update to UI
uploads.append(UploadedFolder(path=collection_path, id=id))
], [])

# send notification to user via email
# SnsClient.get().publish_message(
# profile.push_notification_topic_arn,
# f"DIRT => PlantIT migration completed",
# f"Duration: {str(end - start)}",
# {})

# mark user's profile that DIRT transfer has been completed
end = timezone.now()
profile.dirt_migration_completed = end
profile.save()
user.save()

# push completion update to the UI
async_to_sync(push_migration_event)(user, Migration(
started=start.isoformat(),
completed=None,
completed=end.isoformat(),
num_folders=len(datasets),
target_path=root_collection_path,
downloads=downloads,
uploads=uploads))

# get ID of newly created collection
root_collection_id = client.stat(root_collection_path)['id']

# add collection timestamp as metadata
end = timezone.now()
client.set_metadata(root_collection_id, [
f"dirt_migration_timestamp={end.isoformat()}",
# TODO: anything else we need to add here?
])

# send notification to user via email
SnsClient.get().publish_message(
profile.push_notification_topic_arn,
f"DIRT => PlantIT migration completed",
f"Duration: {str(end - start)}",
{})

# mark user's profile that DIRT transfer has been completed
end = timezone.now()
profile.dirt_migration_completed = end
profile.save()
user.save()

# push completion update to the UI
async_to_sync(push_migration_event)(user, Migration(
started=start.isoformat(),
completed=end.isoformat(),
num_folders=len(datasets),
target_path=root_collection_path,
downloads=downloads,
uploads=uploads))


# see https://stackoverflow.com/a/41119054/6514033
# `@app.on_after_finalize.connect` is necessary for some reason instead of `@app.on_after_configure.connect`
Expand Down
2 changes: 1 addition & 1 deletion plantit/plantit/consumers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def task_event(self, event):
self.logger.info(f"Sending user {self.username} task {task['name']} event (status {task['status']}) to client")
self.send(text_data=json.dumps({'task': task}))

def dirt_migration_event(self, event):
def migration_event(self, event):
migration = event['migration']
self.logger.info(f"DIRT migration status for user {self.username}: {migration}")
self.send(text_data=json.dumps({
Expand Down
Loading

0 comments on commit 9a1da54

Please sign in to comment.