Skip to content

Commit

Permalink
Merge pull request #80 from mytardis/develop
Browse files Browse the repository at this point in the history 
Merging into master for v0.7.0-beta1 build
  • Loading branch information
@wettenhj
wettenhj authored Nov 2, 2016
2 parents 239a150 + 28811b6 commit 8d4d820
Show file tree
Hide file tree
Showing 59 changed files with 1,659 additions and 1,345 deletions.
66 changes: 19 additions & 47 deletions mydata/MyData.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -567,57 +567,29 @@ def OnCloseFrame(self, event):
self.frame.Hide()

# pylint: disable=unused-argument
def ShutDownCleanlyAndExit(self, event):
def ShutDownCleanlyAndExit(self, event, confirm=True):
"""
Shut down MyData cleanly and quit.
"""
started = self.foldersController.Started()
completed = self.foldersController.Completed()
canceled = self.foldersController.Canceled()
failed = self.foldersController.Failed()

message = "Are you sure you want to shut down MyData's " \
"data scans and uploads?"
if started and not completed and not canceled and not failed:
message += "\n\n" \
"MyData will attempt to shut down any uploads currently " \
"in progress."
confirmationDialog = \
wx.MessageDialog(None, message, "MyData",
wx.YES | wx.NO | wx.ICON_QUESTION)
okToExit = confirmationDialog.ShowModal()
okToExit = wx.ID_YES
if confirm and self.Processing():
message = "Are you sure you want to shut down MyData's " \
"data scans and uploads?"
if self.Processing():
message += "\n\n" \
"MyData will attempt to shut down any uploads currently " \
"in progress."
confirmationDialog = \
wx.MessageDialog(None, message, "MyData",
wx.YES | wx.NO | wx.ICON_QUESTION)
okToExit = confirmationDialog.ShowModal()
if okToExit == wx.ID_YES:
def ShutDownDataScansAndUploads():
"""
Shut down data folder scanning, datafile lookups
(verifications) and uploads.
"""
logger.debug("Starting ShutDownDataScansAndUploads...")
# pylint: disable=bare-except
try:
wx.CallAfter(BeginBusyCursorIfRequired)
self.foldersController.ShutDownUploadThreads()
wx.CallAfter(EndBusyCursorIfRequired)
self.tasksModel.ShutDown()
sys.exit(0)
except:
try:
logger.error(traceback.format_exc())
self.tasksModel.ShutDown()
# pylint: disable=protected-access
os._exit(1)
# pylint: disable=bare-except
except:
logger.error(traceback.format_exc())
# pylint: disable=protected-access
os._exit(1)
logger.debug("Finishing run() method for thread %s"
% threading.current_thread().name)

thread = threading.Thread(target=ShutDownDataScansAndUploads)
logger.debug("Starting thread %s" % thread.name)
thread.start()
logger.debug("Started thread %s" % thread.name)
# pylint: disable=bare-except
BeginBusyCursorIfRequired()
self.foldersController.ShutDownUploadThreads()
EndBusyCursorIfRequired()
self.tasksModel.ShutDown()
sys.exit(0)

def OnMinimizeFrame(self, event):
"""
Expand Down
2 changes: 1 addition & 1 deletion mydata/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"""
import sys

__version__ = "0.6.2"
__version__ = "0.7.0-beta1"


try:
Expand Down
16 changes: 9 additions & 7 deletions mydata/controllers/folders.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,14 @@
import Queue
import traceback
import subprocess
import datetime

import requests

import wx
import wx.lib.newevent
import wx.dataview

from mydata.utils.openssh import OPENSSH
from mydata.utils import ConnectionStatus
from mydata.utils import BeginBusyCursorIfRequired
from mydata.utils import EndBusyCursorIfRequired
Expand Down Expand Up @@ -298,6 +299,7 @@ def InitForUploads(self):
fc.SetCompleted(False)
fc.verificationsModel.DeleteAllRows()
fc.uploadsModel.DeleteAllRows()
fc.uploadsModel.SetStartTime(datetime.datetime.now())
fc.verifyDatafileRunnable = {}
fc.verificationsQueue = Queue.Queue()
# For now, the max number of verification threads is hard-coded
Expand Down Expand Up @@ -635,12 +637,6 @@ def ShutDownUploadThreads(self, event=None):
self.verifyDatafileRunnable = {}
self.uploadDatafileRunnable = {}

if sys.platform == 'darwin':
sshControlMasterPool = \
OPENSSH.GetSshControlMasterPool(createIfMissing=False)
if sshControlMasterPool:
sshControlMasterPool.ShutDown()

if self.testRun:
numVerificationsCompleted = \
self.verificationsModel.GetCompletedCount()
Expand Down Expand Up @@ -678,6 +674,12 @@ def ShutDownUploadThreads(self, event=None):
"%d failed upload(s)." % self.uploadsModel.GetFailedCount()
elif self.Completed():
message = "Data scans and uploads completed successfully."
elapsedTime = self.uploadsModel.GetElapsedTime()
if elapsedTime:
averageSpeed = "%3.1f MB/s" % \
(float(self.uploadsModel.GetCompletedSize()) / 1000000.0 \
/ elapsedTime.total_seconds())
message += " Average speed: %s" % averageSpeed
else:
message = "Data scans and uploads appear to have " \
"completed successfully."
Expand Down
5 changes: 2 additions & 3 deletions mydata/controllers/schedule.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def ApplySchedule(self, event, runManually=False,
if scheduleType == "On Startup" and \
self.settingsModel.GetLastSettingsUpdateTrigger() == \
LastSettingsUpdateTrigger.READ_FROM_DISK:
self.CreateOnStartupTask(event)
self.CreateOnStartupTask(event, needToValidateSettings)
elif scheduleType == "On Settings Saved" and \
self.settingsModel.GetLastSettingsUpdateTrigger() == \
LastSettingsUpdateTrigger.UI_RESPONSE:
Expand All @@ -56,7 +56,7 @@ def ApplySchedule(self, event, runManually=False,
self.CreateTimerTask(event)
logger.debug("Finished processing schedule type.")

def CreateOnStartupTask(self, event):
def CreateOnStartupTask(self, event, needToValidateSettings):
"""
Create and schedule task(s) according to the settings configured in
the Schedule tab of the Settings dialog.
Expand All @@ -72,7 +72,6 @@ def OnStartup(event, jobId):
wx.CallAfter(app.DisableTestAndUploadToolbarButtons)
while not app.Processing():
time.sleep(0.01)
needToValidateSettings = False
wx.CallAfter(app.OnRefresh, event, needToValidateSettings,
jobId)
# Sleep this thread until the job is really
Expand Down
63 changes: 27 additions & 36 deletions mydata/controllers/uploads.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import mimetypes
import time
import hashlib
from datetime import datetime

import poster
import requests
Expand All @@ -36,7 +37,6 @@
from mydata.utils.exceptions import ScpException
from mydata.utils.exceptions import IncompatibleMyTardisVersion
from mydata.utils.exceptions import StorageBoxAttributeNotFound
from mydata.utils.exceptions import SshControlMasterLimit

from mydata.logs import logger

Expand Down Expand Up @@ -84,6 +84,8 @@ def Run(self):
self.uploadModel = UploadModel(dataViewId=uploadDataViewId,
folderModel=self.folderModel,
dataFileIndex=self.dataFileIndex)
self.uploadModel.SetExistingUnverifiedDatafile(
self.verificationModel.GetExistingUnverifiedDatafile())
self.uploadsModel.AddRow(self.uploadModel)
self.foldersController.uploadsThreadingLock.release()
self.uploadModel.SetBytesUploadedPreviously(
Expand Down Expand Up @@ -114,10 +116,6 @@ def Run(self):
uploadModel=self.uploadModel))
return

logger.debug("Uploading " +
self.folderModel.GetDataFileName(self.dataFileIndex) +
"...")

if self.foldersController.uploadMethod == UploadMethod.HTTP_POST or \
not self.existingUnverifiedDatafile:
myTardisUrl = self.settingsModel.GetMyTardisUrl()
Expand Down Expand Up @@ -169,10 +167,14 @@ def Md5ProgressCallback(bytesProcessed):
else:
message = "%3d %% MD5 summed" % int(percentComplete)
self.uploadsModel.SetMessage(self.uploadModel, message)
dataFileMd5Sum = \
self.CalculateMd5Sum(dataFilePath, dataFileSize,
self.uploadModel,
progressCallback=Md5ProgressCallback)
if self.settingsModel.FakeMd5Sum():
dataFileMd5Sum = self.settingsModel.GetFakeMd5Sum()
logger.warning("Faking MD5 sum for %s" % dataFilePath)
else:
dataFileMd5Sum = \
self.CalculateMd5Sum(dataFilePath, dataFileSize,
self.uploadModel,
progressCallback=Md5ProgressCallback)

if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
Expand Down Expand Up @@ -235,6 +237,8 @@ def ProgressCallback(current, total, message=None):
if self.uploadModel.Canceled():
self.foldersController.SetCanceled()
return
elif self.uploadModel.GetStatus() == UploadStatus.COMPLETED:
return
if current is None:
# For a zero-sized file, current will be None
# before its upload, and 0 after is upload.
Expand Down Expand Up @@ -284,6 +288,7 @@ def PosterCallback(param, current, total):

message = "Uploading..."
self.uploadsModel.SetMessage(self.uploadModel, message)
self.uploadModel.SetStartTime(datetime.now())
postSuccess = False
uploadSuccess = False

Expand Down Expand Up @@ -329,6 +334,9 @@ def PosterCallback(param, current, total):
# to copy/upload the file to.
tempUrl = response.text
remoteFilePath = tempUrl
dataFileId = \
response.headers['Location'].split('/')[-2]
self.uploadModel.SetDataFileId(dataFileId)
while True:
try:
UploadFile(dataFilePath,
Expand All @@ -340,6 +348,9 @@ def PosterCallback(param, current, total):
self.foldersController,
self.uploadModel)
except IOError, err:
if self.foldersController.IsShuttingDown() or \
self.uploadModel.Canceled():
return
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
Expand All @@ -348,14 +359,14 @@ def PosterCallback(param, current, total):
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
message = "This file will be re-uploaded..."
self.uploadsModel.SetMessage(
self.uploadModel, message)
self.uploadModel.SetProgress(0)
continue
else:
raise
except ScpException, err:
if self.foldersController.IsShuttingDown() or \
self.uploadModel.Canceled():
return
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
Expand All @@ -364,15 +375,14 @@ def PosterCallback(param, current, total):
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
message = \
"This file will be re-uploaded..."
self.uploadsModel.SetMessage(
self.uploadModel, message)
self.uploadModel.SetProgress(0)
continue
else:
raise
except SshException, err:
if self.foldersController.IsShuttingDown() or \
self.uploadModel.Canceled():
return
self.uploadModel.SetTraceback(
traceback.format_exc())
if self.uploadModel.GetRetries() < \
Expand All @@ -381,10 +391,6 @@ def PosterCallback(param, current, total):
self.uploadModel.IncrementRetries()
logger.debug("Restarting upload for " +
dataFilePath)
message = \
"This file will be re-uploaded..."
self.uploadsModel.SetMessage(
self.uploadModel, message)
self.uploadModel.SetProgress(0)
continue
else:
Expand Down Expand Up @@ -520,21 +526,6 @@ def PosterCallback(param, current, total):
message=message,
icon=wx.ICON_ERROR))
return
except SshControlMasterLimit, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController.shutdownUploadsEvent(
failed=True))
message = str(err)
wx.PostEvent(
self.foldersController.notifyWindow,
self.foldersController
.showMessageDialogEvent(title="MyData",
message=message,
icon=wx.ICON_ERROR))
return
except ScpException, err:
self.uploadModel.SetTraceback(
traceback.format_exc())
Expand Down Expand Up @@ -659,11 +650,11 @@ def PosterCallback(param, current, total):
return

if uploadSuccess:
logger.debug("Upload succeeded for " + dataFilePath)
self.uploadsModel.SetStatus(self.uploadModel,
UploadStatus.COMPLETED)
message = "Upload complete!"
self.uploadsModel.SetMessage(self.uploadModel, message)
self.uploadModel.SetLatestTime(datetime.now())
self.uploadModel.SetProgress(100)
self.uploadsModel.UploadProgressUpdated(self.uploadModel)
self.folderModel.SetDataFileUploaded(self.dataFileIndex,
Expand Down
Loading

0 comments on commit 8d4d820

Please sign in to comment.