diff --git a/configs/portraitnet/portraitnet_eg1800_224x224_46k.yml b/configs/portraitnet/portraitnet_eg1800_224x224_46k.yml index 70b9277e5e..fd72f6f854 100644 --- a/configs/portraitnet/portraitnet_eg1800_224x224_46k.yml +++ b/configs/portraitnet/portraitnet_eg1800_224x224_46k.yml @@ -68,5 +68,5 @@ model: backbone: type: MobileNetV2_x1_0 pretrained: https://paddleseg.bj.bcebos.com/dygraph/backbone/mobilenetv2_x1_0_ssld.tar.gz - add_edge: True + add_edge: False num_classes: 2 diff --git a/paddleseg/utils/download.py b/paddleseg/utils/download.py index 5d8bf800ac..8c184f18b9 100644 --- a/paddleseg/utils/download.py +++ b/paddleseg/utils/download.py @@ -146,21 +146,33 @@ def download_file_and_uncompress(url, shutil.rmtree(extraname) full_path = os.path.join(extraname, filename) if filename is not None else extraname + rank_id_curr_node = int(os.environ.get("PADDLE_RANK_IN_NODE", 0)) if not os.path.exists( full_path): # If pretrained model exists, skip download process. - if not os.path.exists(savename): - if not os.path.exists(savepath): - _download_file(url, savepath, print_progress) - - if (not tarfile.is_tarfile(savepath)) and ( - not zipfile.is_zipfile(savepath)): - if not os.path.exists(extraname): - os.makedirs(extraname) - shutil.move(savepath, extraname) - return extraname - - savename = _uncompress_file(savepath, extrapath, delete_file, - print_progress) - savename = os.path.join(extrapath, savename) - shutil.move(savename, extraname) + lock_path = extraname + '.download.lock' + with open(lock_path, 'w'): # touch + os.utime(lock_path, None) + if rank_id_curr_node == 0: + if not os.path.exists(savename): + if not os.path.exists(savepath): + _download_file(url, savepath, print_progress) + + if (not tarfile.is_tarfile(savepath)) and ( + not zipfile.is_zipfile(savepath)): + if not os.path.exists(extraname): + os.makedirs(extraname) + shutil.move(savepath, extraname) + + else: + savename = _uncompress_file(savepath, extrapath, + delete_file, print_progress) + savename = os.path.join(extrapath, savename) + shutil.move(savename, extraname) + + os.remove(lock_path) + + else: + while os.path.exists(lock_path): + time.sleep(0.5) + return extraname diff --git a/paddleseg/utils/utils.py b/paddleseg/utils/utils.py index cad88aacd0..9cf0b9b89f 100644 --- a/paddleseg/utils/utils.py +++ b/paddleseg/utils/utils.py @@ -14,7 +14,6 @@ import os import contextlib -import filelock import tempfile import random from urllib.parse import urlparse, unquote @@ -135,15 +134,14 @@ def download_pretrained_model(pretrained_model): filename = 'model.pdparams' with generate_tempdir() as _dir: - with filelock.FileLock(os.path.join(seg_env.TMP_HOME, savename)): - pretrained_model = download_file_and_uncompress( - pretrained_model, - savepath=_dir, - cover=False, - extrapath=seg_env.PRETRAINED_MODEL_HOME, - extraname=savename, - filename=filename) - pretrained_model = os.path.join(pretrained_model, filename) + pretrained_model = download_file_and_uncompress( + pretrained_model, + savepath=_dir, + cover=False, + extrapath=seg_env.PRETRAINED_MODEL_HOME, + extraname=savename, + filename=filename) + pretrained_model = os.path.join(pretrained_model, filename) return pretrained_model