Fix bookcorpusopen RAM usage (#3280)

* fix bookcorpusopen ram usage * add tags
huggingface · Nov 16, 2021 · ed1b492 · ed1b492 · github-actions · Nov 16, 2021
1 parent 2a64976
commit ed1b492
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 4 deletions.
diff --git a/datasets/bookcorpus/README.md b/datasets/bookcorpus/README.md
@@ -1,10 +1,27 @@
 ---
+annotations_creators:
+- no-annotation
+language_creators:
+- found
 languages:
 - en
+licenses:
+- unknown
+multilinguality:
+- monolingual
+pretty_name: BookCorpus
+size_categories:
+- 10M<n<100M
+source_datasets:
+- original
+task_categories:
+- sequence-modeling
+task_ids:
+- language-modeling
 paperswithcode_id: bookcorpus
 ---
 
-# Dataset Card for "bookcorpus"
+# Dataset Card for BookCorpus
 
 ## Table of Contents
 - [Dataset Description](#dataset-description)

diff --git a/datasets/bookcorpusopen/README.md b/datasets/bookcorpusopen/README.md
@@ -1,11 +1,27 @@
 ---
-pretty_name: BookCorpusOpen
+annotations_creators:
+- no-annotation
+language_creators:
+- found
 languages:
 - en
-paperswithcode_id: null
+licenses:
+- unknown
+multilinguality:
+- monolingual
+pretty_name: BookCorpusOpen
+size_categories:
+- 10K<n<100K
+source_datasets:
+- original
+task_categories:
+- sequence-modeling
+task_ids:
+- language-modeling
+paperswithcode_id: bookcorpus
 ---
 
-# Dataset Card for "bookcorpusopen"
+# Dataset Card for BookCorpusOpen
 
 ## Table of Contents
 - [Dataset Description](#dataset-description)

diff --git a/datasets/bookcorpusopen/bookcorpusopen.py b/datasets/bookcorpusopen/bookcorpusopen.py
@@ -64,6 +64,8 @@ def __init__(self, **kwargs):
 class BookCorpusOpen(datasets.GeneratorBasedBuilder):
     """BookCorpus dataset."""
 
+    DEFAULT_WRITER_BATCH_SIZE = 256  # documents are full books and are quite heavy
+
     BUILDER_CONFIGS = [
         BookCorpusOpenConfig(
             name="plain_text",