albert's comments

huggingface · Oct 8, 2021 · 243ac56 · 243ac56 · github-actions · Oct 8, 2021
1 parent 87e2ab7
commit 243ac56
Show file tree

Hide file tree

Showing 2 changed files with 126 additions and 144 deletions.
diff --git a/src/datasets/data_files.py b/src/datasets/data_files.py
@@ -239,18 +239,20 @@ def __init__(self, data_files: List[Union[Path, Url]], origin_metadata: List[Tup
         super().__init__(data_files)
         self.origin_metadata = origin_metadata
 
-    @staticmethod
+    @classmethod
     def from_hf_repo(
+        cls,
         patterns: List[str],
         dataset_info: huggingface_hub.hf_api.DatasetInfo,
         allowed_extensions: Optional[List[str]] = None,
     ) -> "DataFilesList":
         data_files = resolve_patterns_in_dataset_repository(dataset_info, patterns, allowed_extensions)
         origin_metadata = [(dataset_info.id, dataset_info.sha) for _ in patterns]
-        return DataFilesList(data_files, origin_metadata)
+        return cls(data_files, origin_metadata)
 
-    @staticmethod
+    @classmethod
     def from_local_or_remote(
+        cls,
         patterns: List[str],
         base_path: Optional[str] = None,
         allowed_extensions: Optional[List[str]] = None,
@@ -259,7 +261,7 @@ def from_local_or_remote(
         base_path = base_path if base_path is not None else str(Path().resolve())
         data_files = resolve_patterns_locally_or_by_urls(base_path, patterns, allowed_extensions)
         origin_metadata = _get_origin_metadata_locally_or_by_urls(data_files, use_auth_token=use_auth_token)
-        return DataFilesList(data_files, origin_metadata)
+        return cls(data_files, origin_metadata)
 
 
 class DataFilesDict(Dict[str, DataFilesList]):
@@ -278,14 +280,15 @@ class DataFilesDict(Dict[str, DataFilesList]):
     Changing the order of the keys of this dictionary also doesn't change its hash.
     """
 
-    @staticmethod
+    @classmethod
     def from_local_or_remote(
+        cls,
         patterns: Dict[str, Union[List[str], DataFilesList]],
         base_path: Optional[str] = None,
         allowed_extensions: Optional[List[str]] = None,
         use_auth_token: Optional[Union[bool, str]] = None,
     ) -> "DataFilesDict":
-        out = DataFilesDict()
+        out = cls()
         for key, patterns_for_key in patterns.items():
             out[key] = (
                 DataFilesList.from_local_or_remote(
@@ -299,13 +302,14 @@ def from_local_or_remote(
             )
         return out
 
-    @staticmethod
+    @classmethod
     def from_hf_repo(
+        cls,
         patterns: Dict[str, Union[List[str], DataFilesList]],
         dataset_info: huggingface_hub.hf_api.DatasetInfo,
         allowed_extensions: Optional[List[str]] = None,
     ) -> "DataFilesDict":
-        out = DataFilesDict()
+        out = cls()
         for key, patterns_for_key in patterns.items():
             out[key] = (
                 DataFilesList.from_hf_repo(