use lazy_loader

data_juicer/ops/deduplicator/document_simhash_deduplicator.py

@@ -5,13 +5,13 @@
 from collections import defaultdict, deque
 from typing import Dict, Set
 
+import lazy_loader as lazy
 import numpy as np
 import regex
 from jsonargparse.typing import PositiveInt
 from loguru import logger
 
 from data_juicer.utils.constant import HashKeys
-import lazy_loader as lazy
 
 from ..base_op import AUTOINSTALL, OPERATORS, Deduplicator
 from ..common.helper_func import split_on_whitespace

data_juicer/ops/deduplicator/image_deduplicator.py

@@ -1,10 +1,10 @@
 from collections import defaultdict
 from typing import Dict, Set, Tuple
 
+import lazy_loader as lazy
 import numpy as np
 
 from data_juicer.utils.constant import HashKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import load_data_with_context, load_image
 
 from ..base_op import AUTOINSTALL, OPERATORS, Deduplicator

data_juicer/ops/deduplicator/ray_image_deduplicator.py

@@ -1,7 +1,7 @@
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import PositiveInt
 
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import load_data_with_context, load_image
 
 from ..base_op import AUTOINSTALL, OPERATORS

data_juicer/ops/filter/image_aesthetics_filter.py

@@ -1,9 +1,9 @@
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import ClosedUnitInterval
 from loguru import logger
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import load_data_with_context, load_image
 
 from ...utils.model_utils import get_model, prepare_model

data_juicer/ops/filter/image_face_ratio_filter.py

@@ -1,11 +1,11 @@
 import os
 
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import ClosedUnitInterval
 from loguru import logger
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import (detect_faces, load_data_with_context,
                                         load_image)
 from data_juicer.utils.model_utils import get_model, prepare_model

data_juicer/ops/filter/image_nsfw_filter.py

@@ -1,8 +1,8 @@
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import ClosedUnitInterval
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import load_data_with_context, load_image
 from data_juicer.utils.model_utils import get_model, prepare_model
 

data_juicer/ops/filter/image_text_matching_filter.py

@@ -1,9 +1,9 @@
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import ClosedUnitInterval
 from PIL import ImageOps
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import (SpecialTokens, load_data_with_context,
                                         load_image, remove_special_tokens)
 from data_juicer.utils.model_utils import get_model, prepare_model

data_juicer/ops/filter/image_text_similarity_filter.py

@@ -1,9 +1,9 @@
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import ClosedUnitInterval
 from PIL import ImageOps
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import (SpecialTokens, load_data_with_context,
                                         load_image, remove_special_tokens)
 from data_juicer.utils.model_utils import get_model, prepare_model

data_juicer/ops/filter/image_watermark_filter.py

@@ -1,8 +1,8 @@
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import ClosedUnitInterval
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import load_data_with_context, load_image
 from data_juicer.utils.model_utils import get_model, prepare_model
 

data_juicer/ops/filter/language_id_score_filter.py

@@ -1,10 +1,10 @@
 from typing import List, Tuple, Union
 
+import lazy_loader as lazy
 from jsonargparse.typing import ClosedUnitInterval
 from loguru import logger
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.model_utils import get_model, prepare_model
 
 from ..base_op import AUTOINSTALL, OPERATORS, Filter

data_juicer/ops/filter/perplexity_filter.py

@@ -2,10 +2,10 @@
 # https://huggingface.co/spaces/huggingface/text-data-filtering
 # --------------------------------------------------------
 
+import lazy_loader as lazy
 from jsonargparse.typing import PositiveFloat
 
 from data_juicer.utils.constant import Fields, InterVars, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.model_utils import get_model, prepare_model
 
 from ..base_op import AUTOINSTALL, OPERATORS, Filter

data_juicer/ops/filter/phrase_grounding_recall_filter.py

@@ -1,12 +1,12 @@
 from typing import List
 
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import ClosedUnitInterval
 from loguru import logger
 from PIL import ImageOps
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import (SpecialTokens, iou,
                                         load_data_with_context, load_image,
                                         remove_special_tokens)

data_juicer/ops/filter/stopwords_filter.py

@@ -2,11 +2,11 @@
 # https://huggingface.co/spaces/huggingface/text-data-filtering
 # --------------------------------------------------------
 
+import lazy_loader as lazy
 from jsonargparse.typing import ClosedUnitInterval, List
 
 from data_juicer.utils.asset_utils import ASSET_DIR, load_words_asset
 from data_juicer.utils.constant import Fields, InterVars, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.model_utils import get_model, prepare_model
 
 from ..base_op import AUTOINSTALL, OPERATORS, Filter

data_juicer/ops/filter/token_num_filter.py

@@ -1,9 +1,9 @@
 import sys
 
+import lazy_loader as lazy
 from jsonargparse.typing import PositiveInt
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.model_utils import get_model, prepare_model
 
 from ..base_op import AUTOINSTALL, OPERATORS, Filter

data_juicer/ops/filter/video_aesthetics_filter.py

@@ -1,9 +1,9 @@
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import ClosedUnitInterval, PositiveInt
 from loguru import logger
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import (close_video, extract_key_frames,
                                         extract_video_frames_uniformly,
                                         load_data_with_context, load_video)

data_juicer/ops/filter/video_frames_text_similarity_filter.py

@@ -1,9 +1,9 @@
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import ClosedUnitInterval, PositiveInt
 from PIL import ImageOps
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import (SpecialTokens, close_video,
                                         extract_key_frames,
                                         extract_video_frames_uniformly,

data_juicer/ops/filter/video_motion_score_filter.py

@@ -2,11 +2,11 @@
 from contextlib import contextmanager
 from typing import List, Optional, Sequence, Tuple, Union
 
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import PositiveFloat, PositiveInt
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 
 from ..base_op import AUTOINSTALL, OPERATORS, UNFORKABLE, Filter
 

data_juicer/ops/filter/video_nsfw_filter.py

@@ -1,8 +1,8 @@
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import ClosedUnitInterval, PositiveInt
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import (close_video, extract_key_frames,
                                         extract_video_frames_uniformly,
                                         load_data_with_context, load_video)

data_juicer/ops/filter/video_ocr_area_ratio_filter.py

@@ -1,11 +1,11 @@
 from typing import List, Union
 
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import ClosedUnitInterval, PositiveInt
 
 from data_juicer import cuda_device_count
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import (close_video,
                                         extract_video_frames_uniformly,
                                         load_data_with_context, load_video)

data_juicer/ops/filter/video_watermark_filter.py

@@ -1,8 +1,8 @@
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import ClosedUnitInterval, PositiveInt
 
 from data_juicer.utils.constant import Fields, StatsKeys
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import (close_video, extract_key_frames,
                                         extract_video_frames_uniformly,
                                         load_data_with_context, load_video)

data_juicer/ops/mapper/audio_ffmpeg_wrapped_mapper.py

@@ -1,8 +1,9 @@
 from typing import Dict, List, Optional
 
+import lazy_loader as lazy
+
 from data_juicer.utils.constant import Fields
 from data_juicer.utils.file_utils import transfer_filename
-import lazy_loader as lazy
 from data_juicer.utils.logger_utils import HiddenPrints
 
 from ..base_op import AUTOINSTALL, OPERATORS, Mapper

data_juicer/ops/mapper/image_face_blur_mapper.py

@@ -1,10 +1,10 @@
 import os
 
+import lazy_loader as lazy
 from loguru import logger
 
 from data_juicer.utils.constant import Fields
 from data_juicer.utils.file_utils import transfer_filename
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import (detect_faces, load_data_with_context,
                                         load_image)
 from data_juicer.utils.model_utils import get_model, prepare_model

data_juicer/ops/mapper/nlpaug_en_mapper.py

@@ -1,8 +1,7 @@
 from copy import deepcopy
 
-from loguru import logger
-
 import lazy_loader as lazy
+from loguru import logger
 
 from ..base_op import AUTOINSTALL, OPERATORS, Mapper
 

data_juicer/ops/mapper/nlpcda_zh_mapper.py

@@ -1,8 +1,8 @@
 from copy import deepcopy
 
+import lazy_loader as lazy
 from loguru import logger
 
-import lazy_loader as lazy
 from data_juicer.utils.logger_utils import HiddenPrints
 
 from ..base_op import AUTOINSTALL, OPERATORS, Mapper

data_juicer/ops/mapper/video_face_blur_mapper.py

@@ -1,10 +1,10 @@
 import os
 
 import av
+import lazy_loader as lazy
 
 from data_juicer.utils.constant import Fields
 from data_juicer.utils.file_utils import transfer_filename
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import (close_video, detect_faces,
                                         load_data_with_context, load_video,
                                         process_each_frame)

data_juicer/ops/mapper/video_ffmpeg_wrapped_mapper.py

@@ -1,8 +1,9 @@
 from typing import Dict, List, Optional
 
+import lazy_loader as lazy
+
 from data_juicer.utils.constant import Fields
 from data_juicer.utils.file_utils import transfer_filename
-import lazy_loader as lazy
 from data_juicer.utils.logger_utils import HiddenPrints
 
 from ..base_op import AUTOINSTALL, OPERATORS, Mapper

data_juicer/ops/mapper/video_remove_watermark_mapper.py

@@ -1,12 +1,12 @@
 import os
 
 import av
+import lazy_loader as lazy
 import numpy as np
 from jsonargparse.typing import List, PositiveInt
 
 from data_juicer.utils.constant import Fields
 from data_juicer.utils.file_utils import transfer_filename
-import lazy_loader as lazy
 from data_juicer.utils.logger_utils import HiddenPrints
 from data_juicer.utils.mm_utils import (close_video,
                                         extract_video_frames_uniformly,

data_juicer/ops/mapper/video_resize_aspect_ratio_mapper.py

@@ -2,9 +2,10 @@
 import os
 from fractions import Fraction
 
+import lazy_loader as lazy
+
 from data_juicer.utils.constant import Fields
 from data_juicer.utils.file_utils import transfer_filename
-import lazy_loader as lazy
 from data_juicer.utils.logger_utils import HiddenPrints
 from data_juicer.utils.mm_utils import close_video, load_video
 

data_juicer/ops/mapper/video_resize_resolution_mapper.py

@@ -2,11 +2,11 @@
 import os
 import sys
 
+import lazy_loader as lazy
 from jsonargparse.typing import PositiveInt
 
 from data_juicer.utils.constant import Fields
 from data_juicer.utils.file_utils import transfer_filename
-import lazy_loader as lazy
 from data_juicer.utils.logger_utils import HiddenPrints
 from data_juicer.utils.mm_utils import close_video, load_video
 

data_juicer/ops/mapper/video_split_by_scene_mapper.py

@@ -2,12 +2,12 @@
 import re
 from itertools import chain
 
+import lazy_loader as lazy
 from jsonargparse.typing import NonNegativeFloat, NonNegativeInt
 
 from data_juicer.utils.constant import Fields
 from data_juicer.utils.file_utils import (add_suffix_to_filename,
                                           transfer_filename)
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import SpecialTokens
 
 from ..base_op import AUTOINSTALL, OPERATORS, Mapper

data_juicer/ops/mapper/video_tagging_from_audio_mapper.py

@@ -1,7 +1,7 @@
+import lazy_loader as lazy
 import librosa
 
 from data_juicer.utils.constant import Fields
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import extract_audio_from_video
 from data_juicer.utils.model_utils import get_model, prepare_model
 

data_juicer/ops/mapper/video_tagging_from_frames_mapper.py

@@ -1,9 +1,9 @@
 from collections import Counter
 
+import lazy_loader as lazy
 from jsonargparse.typing import PositiveInt
 
 from data_juicer.utils.constant import Fields
-import lazy_loader as lazy
 from data_juicer.utils.mm_utils import (close_video, extract_key_frames,
                                         extract_video_frames_uniformly,
                                         load_data_with_context, load_video)