From a4f1d15ff2c624d6dbc5340b913a2ce4ed082918 Mon Sep 17 00:00:00 2001 From: BPCZ <66191364+BPCZ@users.noreply.github.com> Date: Fri, 4 Dec 2020 17:22:36 -0800 Subject: [PATCH] increases possible size of csv fields (#67) Co-authored-by: BPCZ --- aitextgen/TokenDataset.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/aitextgen/TokenDataset.py b/aitextgen/TokenDataset.py index a0cb985..3aa9b4d 100644 --- a/aitextgen/TokenDataset.py +++ b/aitextgen/TokenDataset.py @@ -10,6 +10,9 @@ import itertools from tqdm.auto import tqdm import numpy as np +import sys + +csv.field_size_limit(sys.maxsize) logger = logging.getLogger(__name__) logger.setLevel(logging.INFO)