Don't compute checksums if not necessary in datasets-cli test (#5603)

don't compute checksums if not necessary
huggingface · Mar 3, 2023 · 24fb01b · 24fb01b
1 parent 6569014
commit 24fb01b
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/datasets/commands/test.py b/src/datasets/commands/test.py
@@ -140,7 +140,9 @@ def get_builders() -> Generator[DatasetBuilder, None, None]:
 
         for j, builder in enumerate(get_builders()):
             print(f"Testing builder '{builder.config.name}' ({j + 1}/{n_builders})")
-            builder._record_infos = True
+            builder._record_infos = os.path.exists(
+                os.path.join(builder.get_imported_module_dir(), datasets.config.DATASETDICT_INFOS_FILENAME)
+            )  # record checksums only if we need to update a (deprecated) dataset_infos.json
             builder.download_and_prepare(
                 download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
                 if not self._force_redownload