Skip to content

Commit

Permalink
Fix cache conflict in _check_legacy_cache2 (#6792)
Browse files Browse the repository at this point in the history
fix cache conflict in _check_legacy_cache2
  • Loading branch information
lhoestq authored Apr 9, 2024
1 parent 3575036 commit 0f27d7b
Showing 1 changed file with 6 additions and 1 deletion.
7 changes: 6 additions & 1 deletion src/datasets/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -370,6 +370,7 @@ def __init__(
config_kwargs["data_files"] = data_files
if data_dir is not None:
config_kwargs["data_dir"] = data_dir
self.config_kwargs = config_kwargs
self.config, self.config_id = self._create_builder_config(
config_name=config_name,
custom_features=features,
Expand Down Expand Up @@ -488,7 +489,11 @@ def _check_legacy_cache(self) -> Optional[str]:

def _check_legacy_cache2(self, dataset_module: "DatasetModule") -> Optional[str]:
"""Check for the old cache directory template {cache_dir}/{namespace}___{dataset_name}/{config_name}-xxx from 2.14 and 2.15"""
if self.__module__.startswith("datasets.") and not is_remote_url(self._cache_dir_root):
if (
self.__module__.startswith("datasets.")
and not is_remote_url(self._cache_dir_root)
and not (set(self.config_kwargs) - {"data_files", "data_dir"})
):
from .packaged_modules import _PACKAGED_DATASETS_MODULES
from .utils._dill import Pickler

Expand Down

0 comments on commit 0f27d7b

Please sign in to comment.