Retry upload on failure

mytardis · Feb 19, 2021 · f838f20 · f838f20
1 parent aebac2f
commit f838f20
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 8 deletions.
diff --git a/mydata/__init__.py b/mydata/__init__.py
@@ -8,7 +8,7 @@
 import subprocess
 import sys
 
-__version__ = "0.9.4"
+__version__ = "0.9.5"
 
 
 if hasattr(sys, "frozen"):

diff --git a/mydata/utils/openssh.py b/mydata/utils/openssh.py
@@ -494,7 +494,8 @@ def UploadFile(filePath, fileSize, username, privateKeyFilePath,
                 SETTINGS.general.apiKey,
                 filePath,
                 uploadModel,
-                progressCallback)
+                progressCallback,
+                SETTINGS.advanced.maxUploadRetries)
         except Exception as err:
             raise UploadFailed(err)
 

diff --git a/mydata/utils/upload.py b/mydata/utils/upload.py
@@ -13,6 +13,7 @@
 
 from ssh2 import session, sftp
 
+from ..logs import logger
 from ..models.datafile import DataFileModel
 
 
@@ -36,6 +37,7 @@ def HandleResponse(rsp):
     try:
         data = json.loads(rsp.content)
     except:
+        logger.error("Server response:\n{}".format(rsp.content))
         raise Exception("Unexpected data received from the server.")
     if "success" not in data:
         if "error_message" in data:
@@ -110,7 +112,8 @@ def GetDataFileObjectId(uploadModel):
     return None
 
 def UploadFileChunked(server, username, apiKey,
-                      filePath, uploadModel, progressCallback):
+                      filePath, uploadModel, progressCallback,
+                      maxUploadRetries):
     """
     Upload file using chunks API
     """
@@ -128,16 +131,27 @@ def UploadFileChunked(server, username, apiKey,
         file = open(filePath, "rb")
         for thisChunk in range(math.ceil(fileSize/status["size"])):
             thisOffset = thisChunk*status["size"]
+            currentRetry = 0
             if thisOffset >= totalUploaded:
                 file.seek(thisOffset)
                 binaryData = file.read(status["size"])
                 backoffSleep = DefaultSleepIdle()
                 while backoffSleep > 0 and not uploadModel.canceled:
-                    upload = UploadChunk(
-                        server, username, apiKey, uploadModel.dfoId,
-                        status["checksum"],
-                        "%s-%s/%s" % (thisOffset, thisOffset+len(binaryData), fileSize),
-                        binaryData)
+                    try:
+                        currentRetry += 1
+                        upload = UploadChunk(
+                            server, username, apiKey, uploadModel.dfoId,
+                            status["checksum"],
+                            "%s-%s/%s" % (thisOffset, thisOffset+len(binaryData), fileSize),
+                            binaryData)
+                    except Exception as e:
+                        if currentRetry <= maxUploadRetries:
+                            logger.error("Can't upload chunk: {}".format(str(e)))
+                            upload = {
+                                "success": False
+                            }
+                        else:
+                            raise Exception(str(e))
                     if not upload["success"]:
                         sleep(backoffSleep)
                         backoffSleep *= 2