preprocess.py

#################################################
### THIS FILE WAS AUTOGENERATED! DO NOT EDIT! ###
#################################################
# file to edit: dev_nb/preprocess.ipynb
from fastai import *
from fastai.text import *

path = Path('../data')

def preprocess(data,bs=64):
    if data == 'gutenberg':
        data_lm = (TextList.from_folder(path/'gutenberg',extensions=".txt",recurse=False)
                    .split_by_rand_pct(0.1)
                    .label_for_lm()
                    .databunch(bs=bs))
        data_lm.save('gutenberg_tmp')
    elif data == 'lyrics':
        df = pd.read_json(path/'lyrics.json',lines=True)
        df['text'] = df['text'].astype('str')
        data_lm = (TextList.from_df(df,cols='text',path=path)
                   .random_split_by_pct(0.1)
                   .label_for_lm()
                   .databunch(bs=bs))
        data_lm.save('tmp_lyrics')
    elif data == 'poems':
        poems = (path/'poems.txt').open().read().split('\n\n\n\n\n\n\n')
        poems_df = pd.DataFrame(poems)
        tokenizer = Tokenizer(SpacyTokenizer, 'en')
        processor = [TokenizeProcessor(tokenizer=tokenizer), NumericalizeProcessor(min_freq=1,max_vocab=60000)]
        data_lm = (TextList.from_df(poems_df,path,cols=0,processor=processor)
                    .split_by_rand_pct(0.1)
                    .label_for_lm()
                    .databunch(bs=bs))
        data_lm.save('poems_tmp')
    elif data == 'metaphors':
        df = pd.read_csv(path/'metaphors.csv')
        df['metaphor'] = df['metaphor'].astype('str')
        metaphors = list(df['metaphor'])
        cleaned_metaphors = []
        for metaphor in metaphors:
            metaphor = metaphor.replace('/ ','').replace('"','').replace("'","").replace('<i> ','').replace(' </i>',"")
            cleaned_metaphors.append(metaphor)
        df_met = pd.DataFrame(cleaned_metaphors)
        tokenizer = Tokenizer(SpacyTokenizer, 'en')
        processor = [TokenizeProcessor(tokenizer=tokenizer), NumericalizeProcessor(min_freq=1,max_vocab=50000)]
        data_lm = (TextList.from_df(df_met,cols=0,path=path,processor=processor)
                           .random_split_by_pct(0.1)
                           .label_for_lm()
                           .databunch(bs=bs))
        data_lm.save('tmp_metaphors')
    else: print(f'data {data} not supported')