Nextflow - default protocol (file); some globbing #19

elixir-cloud-aai · Aug 18, 2020 · 28090a7 · 28090a7
2 parents 47e79a7 + d59bd69
commit 28090a7
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 7 deletions.
diff --git a/src/tesk_core/filer.py b/src/tesk_core/filer.py
@@ -13,6 +13,7 @@
 import requests
 from tesk_core.exception import UnknownProtocol, FileProtocolDisabled
 import shutil
+from glob import glob
 from tesk_core.path import containerPath, getPath, fileEnabled
 
 try:
@@ -157,6 +158,21 @@ def copyDir(src, dst):
 
         shutil.copytree(src, dst)
 
+def copyFile(src, dst):
+    '''
+    Limitations of shutil.copy:
+
+    It does not interpret * as a glob, but as a character.
+    '''
+
+    # If there is any * in 'dst', use only the dirname (base path)
+    p = re.compile('.*\*.*')
+    if p.match(dst):
+        dst=os.path.dirname(dst)
+
+    for file in glob(src):
+        shutil.copy(file, dst)
+
 
 class FileTransput(Transput):
     def __init__(self, path, url, ftype):
@@ -172,7 +188,7 @@ def transfer(self, copyFn, src, dst):
 
     def download_file(self): self.transfer(shutil.copy  , self.urlContainerPath , self.path)
     def download_dir(self):  self.transfer(copyDir      , self.urlContainerPath , self.path)
-    def upload_file(self):   self.transfer(shutil.copy  , self.path             , self.urlContainerPath)        
+    def upload_file(self):   self.transfer(copyFile  , self.path             , self.urlContainerPath)
     def upload_dir(self):    self.transfer(copyDir      , self.path             , self.urlContainerPath)        
 
 
@@ -454,8 +470,8 @@ def process_file(ttype, filedata):
 
     scheme = urlparse(filedata['url']).scheme
     if scheme == '':
-        logging.error('Could not determine protocol for url: "%s"', filedata['url'])
-        return 1
+        logging.info('Could not determine protocol for url: "%s", assuming "file"', filedata['url'])
+        scheme='file'
 
     trans = newTransput(scheme)
 

diff --git a/tests/test_filer.py b/tests/test_filer.py
@@ -1,6 +1,6 @@
 import unittest
 from tesk_core.filer import newTransput, FTPTransput, HTTPTransput, FileTransput,\
-    process_file, logConfig, getPath, copyDir
+    process_file, logConfig, getPath, copyDir, copyFile
 from tesk_core.exception import UnknownProtocol, InvalidHostPath,\
     FileProtocolDisabled
 from tesk_core.path import containerPath
@@ -94,8 +94,8 @@ def test_upload_dir(self, copyMock, copyDirMock):
                                             , '/transfer/tmphrtip1o8')
 
     @patch('tesk_core.filer.copyDir')
-    @patch('tesk_core.filer.shutil.copy')
-    def test_upload_file(self, copyMock, copyDirMock):
+    @patch('tesk_core.filer.copyFile')
+    def test_upload_file(self, copyFileMock, copyDirMock):
 
         filedata = {
             "url": "file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/md5",
@@ -108,10 +108,29 @@ def test_upload_file(self, copyMock, copyDirMock):
 
         copyDirMock.assert_not_called()
 
-        copyMock.assert_called_once_with( '/TclSZU/md5'
+        copyFileMock.assert_called_once_with( '/TclSZU/md5'
                                         , '/transfer/tmphrtip1o8/md5')
 
 
+    @patch('tesk_core.filer.copyDir')
+    @patch('tesk_core.filer.copyFile')
+    def test_upload_file_glob(self, copyFileMock, copyDirMock):
+
+        filedata = {
+            "url": "file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/md5*",
+            "path": "/TclSZU/md5*",
+            "type": "FILE",
+            "name": "stdout"
+        }
+
+        process_file('outputs', filedata)
+
+        copyDirMock.assert_not_called()
+
+        copyFileMock.assert_called_once_with( '/TclSZU/md5*'
+                                        , '/transfer/tmphrtip1o8/md5*')
+
+
     def test_copyDir(self):
 
         def rmDir(d):
@@ -170,6 +189,11 @@ def test_getPath(self):
         self.assertEquals( getPath('file:///home/tfga/workspace/cwl-tes/tmphrtip1o8/md5')
                          ,                '/home/tfga/workspace/cwl-tes/tmphrtip1o8/md5')
 
+    def test_getPathNoScheme(self):
+
+        self.assertEquals( getPath('/home/tfga/workspace/cwl-tes/tmphrtip1o8/md5')
+                         ,         '/home/tfga/workspace/cwl-tes/tmphrtip1o8/md5')
+
 
     def test_containerPath(self):