Add an option to use rclone copy with sync

mlcommons · Mar 15, 2024 · 2ce17a7 · 2ce17a7
1 parent 93a522e
commit 2ce17a7
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/cm-mlops/script/download-file/_cm.json b/cm-mlops/script/download-file/_cm.json
@@ -10,7 +10,9 @@
       "tags": "detect,os"
     }
   ],
-  "env": {},
+  "default_env": {
+    "CM_RCLONE_COPY_USING": "sync"
+  },
   "input_description": {},
   "input_mapping": {
     "download_path": "CM_DOWNLOAD_PATH",

diff --git a/cm-mlops/script/download-file/customize.py b/cm-mlops/script/download-file/customize.py
@@ -112,7 +112,8 @@ def preprocess(i):
         elif tool == "rclone":
             if env.get('CM_RCLONE_CONFIG_CMD', '') != '':
                 env['CM_DOWNLOAD_CONFIG_CMD'] = env['CM_RCLONE_CONFIG_CMD']
-            env['CM_DOWNLOAD_CMD'] = f"rclone sync {url} {os.path.join(os.getcwd(), env['CM_DOWNLOAD_FILENAME'])} -P"
+            rclone_copy_using = env.get('CM_RCLONE_COPY_USING', 'sync')
+            env['CM_DOWNLOAD_CMD'] = f"rclone {rclone_copy_using} {url} {os.path.join(os.getcwd(), env['CM_DOWNLOAD_FILENAME'])} -P"
 
         filename = env['CM_DOWNLOAD_FILENAME']
         env['CM_DOWNLOAD_DOWNLOADED_FILENAME'] = filename