Title prefix format: both = title = and = title = are OK (#186)

Korpora/korpus_kowiki.py

@@ -72,7 +72,7 @@ def __init__(self, root_dir=None, force_download=False):
     def load(self, path, num_lines=-1):
         def split_title_text(wikitext):
             lines = wikitext.split('\n')
-            title = lines[0]
+            title = lines[0].strip()
             text = '\n'.join([line.strip() for line in lines[2:] if line.strip()])
             return title, text
 

Korpora/utils.py

@@ -77,17 +77,18 @@ def load_wikitext(path, num_lines=-1):
     """
     if num_lines <= 0:
         with open(path, encoding='utf-8') as f:
-            texts = f.read().split('\n =')
+            # noise robust
+            texts = f.read().replace('\n =', '\n=').split('\n=')
     else:
         lines = []
         with open(path, encoding='utf-8') as f:
             for i, line in enumerate(f):
                 if (i >= num_lines):
                     break
-                lines.append(line)
-        texts = ''.join(lines).split('\n =')
+                lines.append(line.strip() + "\n")
+        texts = ''.join(lines).split('\n=')
     # fix missing prefix
-    texts = [texts[0]] + [f' ={text}' for text in texts[1:]]
+    texts = [texts[0]] + [f'={text}' for text in texts[1:]]
     return texts