increases possible size of csv fields (#67)

Co-authored-by: BPCZ <BPCZ@protonmail.com>
minimaxir · Dec 5, 2020 · a4f1d15 · a4f1d15
1 parent f7278bf
commit a4f1d15
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/aitextgen/TokenDataset.py b/aitextgen/TokenDataset.py
@@ -10,6 +10,9 @@
 import itertools
 from tqdm.auto import tqdm
 import numpy as np
+import sys
+
+csv.field_size_limit(sys.maxsize)
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)