Skip to content

Commit

Permalink
Do not use Parquet exports if revision is passed (#6555)
Browse files Browse the repository at this point in the history
* Do not use Parquet exports if revision is passed

* Remove unused revision
  • Loading branch information
albertvillanova authored Feb 2, 2024
1 parent f9975f6 commit 8f22ec7
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions src/datasets/load.py
Original file line number Diff line number Diff line change
Expand Up @@ -1816,10 +1816,10 @@ def dataset_module_factory(
raise e
if filename in [sibling.rfilename for sibling in dataset_info.siblings]: # contains a dataset script
fs = HfFileSystem(endpoint=config.HF_ENDPOINT, token=download_config.token)
if _require_custom_configs:
if _require_custom_configs or (revision and revision != "main"):
can_load_config_from_parquet_export = False
elif _require_default_config_name:
with fs.open(f"datasets/{path}/{filename}", "r", revision=revision, encoding="utf-8") as f:
with fs.open(f"datasets/{path}/{filename}", "r", encoding="utf-8") as f:
can_load_config_from_parquet_export = "DEFAULT_CONFIG_NAME" not in f.read()
else:
can_load_config_from_parquet_export = True
Expand Down

0 comments on commit 8f22ec7

Please sign in to comment.