[cherry-pick] nranks bug of download

configs/portraitnet/portraitnet_eg1800_224x224_46k.yml

@@ -68,5 +68,5 @@ model:
   backbone:
     type: MobileNetV2_x1_0
     pretrained: https://paddleseg.bj.bcebos.com/dygraph/backbone/mobilenetv2_x1_0_ssld.tar.gz
-  add_edge: True
+  add_edge: False
   num_classes: 2

paddleseg/utils/download.py

@@ -146,21 +146,33 @@ def download_file_and_uncompress(url,
             shutil.rmtree(extraname)
     full_path = os.path.join(extraname,
                              filename) if filename is not None else extraname
+    rank_id_curr_node = int(os.environ.get("PADDLE_RANK_IN_NODE", 0))
     if not os.path.exists(
             full_path):  # If pretrained model exists, skip download process.
-        if not os.path.exists(savename):
-            if not os.path.exists(savepath):
-                _download_file(url, savepath, print_progress)
-
-            if (not tarfile.is_tarfile(savepath)) and (
-                    not zipfile.is_zipfile(savepath)):
-                if not os.path.exists(extraname):
-                    os.makedirs(extraname)
-                shutil.move(savepath, extraname)
-                return extraname
-
-            savename = _uncompress_file(savepath, extrapath, delete_file,
-                                        print_progress)
-            savename = os.path.join(extrapath, savename)
-        shutil.move(savename, extraname)
+        lock_path = extraname + '.download.lock'
+        with open(lock_path, 'w'):  # touch    
+            os.utime(lock_path, None)
+        if rank_id_curr_node == 0:
+            if not os.path.exists(savename):
+                if not os.path.exists(savepath):
+                    _download_file(url, savepath, print_progress)
+
+                if (not tarfile.is_tarfile(savepath)) and (
+                        not zipfile.is_zipfile(savepath)):
+                    if not os.path.exists(extraname):
+                        os.makedirs(extraname)
+                    shutil.move(savepath, extraname)
+
+                else:
+                    savename = _uncompress_file(savepath, extrapath,
+                                                delete_file, print_progress)
+                    savename = os.path.join(extrapath, savename)
+                    shutil.move(savename, extraname)
+
+            os.remove(lock_path)
+
+        else:
+            while os.path.exists(lock_path):
+                time.sleep(0.5)
+
     return extraname

paddleseg/utils/utils.py

@@ -14,7 +14,6 @@
 
 import os
 import contextlib
-import filelock
 import tempfile
 import random
 from urllib.parse import urlparse, unquote
@@ -135,15 +134,14 @@ def download_pretrained_model(pretrained_model):
         filename = 'model.pdparams'
 
     with generate_tempdir() as _dir:
-        with filelock.FileLock(os.path.join(seg_env.TMP_HOME, savename)):
-            pretrained_model = download_file_and_uncompress(
-                pretrained_model,
-                savepath=_dir,
-                cover=False,
-                extrapath=seg_env.PRETRAINED_MODEL_HOME,
-                extraname=savename,
-                filename=filename)
-            pretrained_model = os.path.join(pretrained_model, filename)
+        pretrained_model = download_file_and_uncompress(
+            pretrained_model,
+            savepath=_dir,
+            cover=False,
+            extrapath=seg_env.PRETRAINED_MODEL_HOME,
+            extraname=savename,
+            filename=filename)
+        pretrained_model = os.path.join(pretrained_model, filename)
     return pretrained_model