Recursively put files sometimes

.travis.yml

@@ -1,7 +1,12 @@
 sudo: required
 language: java
+services:
+  - docker
 jdk:
   - oraclejdk8
+install:
+  - pip install --user -r requirements.txt
+before_script:
+  - bash start-localstack.sh
 script: 
   - mvn clean install
-

requirements.txt

@@ -0,0 +1,3 @@
+pip==9.0.1
+localstack==0.8.5
+s3cmd==2.0.1

src/main/java/io/dockstore/provision/S3CmdPlugin.java

@@ -174,6 +174,15 @@ public boolean uploadTo(String destPath, Path sourceFile, Optional<String> metad
             } catch (IOException e) {
                 throw new RuntimeException(e);
             }
+            String recursive = "";
+
+            // If the destination end with a slash, source could be either a file or a directory
+            // It can technically be either, but we're recursively putting the source file as if it's a directory because there's no side effect
+            // If the destination does not end with a slash, source must be a file
+            if (destPath.endsWith("/")) {
+                recursive = "-r ";
+            }
+
             String modifiedChunkSize = getChunkSize(sizeInBytes);
             destPath = destPath.replace("s3cmd://", "s3://");
             String trimmedPath = destPath.replace("s3://", "");
@@ -189,7 +198,7 @@ public boolean uploadTo(String destPath, Path sourceFile, Optional<String> metad
                     LOG.error("Could not create bucket");
                 }
             }
-            String command = client + " -c " + configLocation + " put " + sourceFile.toString().replace(" ", "%32") + " " + destPath
+            String command = client + " -c " + configLocation + " put " + recursive + sourceFile.toString().replace(" ", "%32") + " " + destPath
                     + modifiedChunkSize;
             int exitCode = executeConsoleCommand(command, true);
             return checkExitCode(exitCode);

src/test/java/io/dockstore/provision/S3CmdPluginTest.java

@@ -0,0 +1,87 @@
+package io.dockstore.provision;
+
+import java.io.File;
+import java.lang.reflect.Method;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author gluu
+ * @since 07/03/17
+ */
+public class S3CmdPluginTest {
+    S3CmdPlugin.S3CmdProvision icgcGetProvision;
+    final String resourcesDirectory = new File("src/test/resources").getAbsolutePath();
+    @Before
+    public void before() throws Exception {
+        icgcGetProvision = new S3CmdPlugin.S3CmdProvision();
+        setConfiguration();
+    }
+
+    private void setConfiguration() throws Exception {
+        HashMap hm = new HashMap();
+        hm.put("client", "/home/travis/.local/bin/s3cmd");
+//        hm.put("client", "/usr/local/bin/s3cmd");
+        hm.put("config-file-location", resourcesDirectory + "/.s3cfg");
+        hm.put("verbosity", "Minimal");
+        icgcGetProvision.setConfiguration(hm);
+    }
+
+    private void download(String source, String destination) {
+        String sourcePath = source;
+        File f = new File(destination);
+        assertFalse(f.getAbsolutePath() + " already exists.",f.exists());
+        Path destinationPath = Paths.get(f.getAbsolutePath());
+        assertTrue(icgcGetProvision.downloadFrom(sourcePath, destinationPath));
+        assertTrue(f.getAbsolutePath() + " not downloaded." , f.exists());
+    }
+
+    /**
+     * This tests if a file can be uploaded to another file and then downloaded as a different file
+     */
+    @Test
+    public void uploadFileToFile() {
+        String destPath = "s3cmd://test-bucket1/file2.txt";
+        Path sourceFile = Paths.get(resourcesDirectory + "/inputFilesDirectory/file.txt");
+        icgcGetProvision.uploadTo(destPath, sourceFile, null);
+        String source = "s3cmd://test-bucket1/file2.txt";
+        String destination = resourcesDirectory +"/outputFilesDirectory1/file3.txt";
+        download(source, destination);
+    }
+
+    /**
+     * This tests if a file can be uploaded to a directory then downloaded as a different file
+     */
+    @Test
+    public void uploadFileToDirectory() {
+        String destPath = "s3cmd://test-bucket2/";
+        Path sourceFile = Paths.get(resourcesDirectory + "/inputFilesDirectory/file.txt");
+        icgcGetProvision.uploadTo(destPath, sourceFile, null);
+        String source = "s3cmd://test-bucket2/file.txt";
+        String destination = resourcesDirectory +"/outputFilesDirectory2/file2.txt";
+        download(source, destination);
+    }
+
+    /**
+     * This tests if a directory can be uploaded to another directory and then all of its files could be downloaded
+     */
+    @Test
+    public void uploadDirectoryToDirectory() {
+        String destPath = "s3cmd://test-bucket3/";
+        Path sourceFile = Paths.get(resourcesDirectory + "/inputFilesDirectory");
+        icgcGetProvision.uploadTo(destPath, sourceFile, null);
+        String source = "s3cmd://test-bucket3/inputFilesDirectory/file.txt";
+        String destination = resourcesDirectory +"/outputFilesDirectory3/file.txt";
+        download(source, destination);
+        source = "s3cmd://test-bucket3/inputFilesDirectory/file2.txt";
+        destination = resourcesDirectory +"/outputFilesDirectory3/file2.txt";
+        download(source, destination);
+    }
+}

src/test/java/io/dockstore/provision/VerbosityEnumTest.java

@@ -0,0 +1,25 @@
+package io.dockstore.provision;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author gluu
+ * @since 14/02/18
+ */
+public class VerbosityEnumTest {
+    /**
+     * This tests if the enum can be converted to the correct integer
+     */
+    @Test
+    public void getLevel() throws Exception {
+        Assert.assertEquals(1 ,VerbosityEnum.QUIET.getLevel());
+        Assert.assertEquals(2 ,VerbosityEnum.MINIMAL.getLevel());
+        Assert.assertEquals(3 ,VerbosityEnum.NORMAL.getLevel());
+        Assert.assertEquals(4 ,VerbosityEnum.DETAILED.getLevel());
+        Assert.assertEquals(5 ,VerbosityEnum.DIAGNOSTIC.getLevel());
+    }
+
+}

src/test/resources/.s3cfg

@@ -0,0 +1,76 @@
+[default]
+access_key = potato
+access_token =
+add_encoding_exts =
+add_headers =
+bucket_location = potato
+ca_certs_file =
+cache_file =
+check_ssl_certificate = True
+check_ssl_hostname = True
+cloudfront_host = cloudfront.amazonaws.com
+default_mime_type = binary/octet-stream
+delay_updates = False
+delete_after = False
+delete_after_fetch = False
+delete_removed = False
+dry_run = False
+enable_multipart = True
+encoding = UTF-8
+encrypt = False
+expiry_date =
+expiry_days =
+expiry_prefix =
+follow_symlinks = False
+force = False
+get_continue = False
+gpg_command = /usr/bin/gpg
+gpg_decrypt = %(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_encrypt = %(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s
+gpg_passphrase =
+guess_mime_type = True
+host_base = localhost:4572
+host_bucket =
+human_readable_sizes = False
+invalidate_default_index_on_cf = False
+invalidate_default_index_root_on_cf = True
+invalidate_on_cf = False
+kms_key =
+limit = -1
+limitrate = 0
+list_md5 = False
+log_target_prefix =
+long_listing = False
+max_delete = -1
+mime_type =
+multipart_chunk_size_mb = 15
+multipart_max_chunks = 10000
+preserve_attrs = True
+progress_meter = True
+proxy_host =
+proxy_port = 0
+put_continue = False
+recursive = False
+recv_chunk = 65536
+reduced_redundancy = False
+requester_pays = False
+restore_days = 1
+restore_priority = Standard
+secret_key =
+send_chunk = 65536
+server_side_encryption = False
+signature_v2 = False
+simpledb_host = sdb.amazonaws.com
+skip_existing = False
+socket_timeout = 300
+stats = False
+stop_on_error = False
+storage_class =
+urlencoding_mode = normal
+use_http_expect = False
+use_https = False
+use_mime_magic = True
+verbosity = WARNING
+website_endpoint = http://%(bucket)s.s3-website-%(location)s.amazonaws.com/
+website_error =
+website_index = index.html

src/test/resources/inputFilesDirectory/file.txt

@@ -0,0 +1 @@
+asdf

src/test/resources/inputFilesDirectory/file2.txt

@@ -0,0 +1 @@
+asdf

start-localstack.sh

@@ -0,0 +1,6 @@
+localstack start > localstack.log &
+while ! less localstack.log | grep Ready. ; do
+  sleep 1
+done
+echo 'done'
+