Data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /Users/june/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /Users/june/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from glob2 import glob\n",
    "import json\n",
    "\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "import re\n",
    "from nltk.corpus import stopwords as stpdfa\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "import gensim\n",
    "import gensim.models.word2vec as w2v\n",
    "from gensim.test.utils import common_texts\n",
    "from gensim.models.doc2vec import Doc2Vec, TaggedDocument\n",
    "from datetime import datetime\n",
    "from datetime import date\n",
    "from datetime import timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', 400)\n",
    "# pd.set_option('display.max_rowwidth', 100)\n",
    "pd.set_option('display.max_colwidth', 400)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TOC\n",
    "1. Data Import\n",
    "    1. Feature Extraction\n",
    "2. Text Vectorization\n",
    "    2. Word2Vec\n",
    "    3. Doc2Vec\n",
    "4. Data Export as CSV files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Import"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "def cross_val_jsons(jsonFiles, isTest):\n",
    "    # output = [json for json in jsonFiles]\n",
    "    # print(random.choice(jsonFiles))\n",
    "    if (isTest == True):\n",
    "        test = random.choice(jsonFiles)\n",
    "        jsonFiles.remove(test)\n",
    "        train = [json for event in jsonFiles for json in event]\n",
    "        return train, test\n",
    "    else:\n",
    "        data = [json for event in jsonFiles for json in event]\n",
    "        return [data]\n",
    "\n",
    "def extract_data(datas):\n",
    "    data_lists = []\n",
    "    isRumorLists = []\n",
    "    for index, dataset in enumerate(datas):\n",
    "        data_list = []\n",
    "        isRumorList = []\n",
    "        count = 0 # help var\n",
    "\n",
    "        for jsonFile in dataset:\n",
    "            count+=1\n",
    "            if jsonFile.find(\"non-rumours\") == -1:\n",
    "                isRumorList.append(1)\n",
    "            else:\n",
    "                isRumorList.append(0)\n",
    "\n",
    "            with open (jsonFile, 'r') as f:\n",
    "                for l in f.readlines():\n",
    "                    if not l.strip (): # skip empty lines\n",
    "                        continue\n",
    "                    json_data = json.loads(l)\n",
    "                    # print (json_data,\"\\n\\n\")\n",
    "                    data_list.append(json_data)\n",
    "\n",
    "        isRumorLists.append(pd.DataFrame(isRumorList, columns=['isRumor']))\n",
    "        data_lists.append(data_list)\n",
    "    return data_lists, isRumorLists\n",
    "\n",
    "def printRumor(route):\n",
    "    if route.find(\"rumours\") == -1:\n",
    "        print('non-rumors')\n",
    "    else:\n",
    "        print('rumor')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def capitalratio(tweet_text):\n",
    "    uppers = [l for l in tweet_text if l.isupper()]\n",
    "    capitalratio = len(uppers) / len(tweet_text)\n",
    "    return capitalratio \n",
    "\n",
    "def tweets2tokens(tweet_text):\n",
    "    # Tokenizing\n",
    "    urls = []\n",
    "    tokens = nltk.word_tokenize(re.sub(r'([^\\s\\w]|_)+','', tweet_text.lower()))\n",
    "    tweet_text = re.sub(r\"http\\S+\", \"\", tweet_text)\n",
    "    # tokens = nltk.TweetTokenizer().tokenize(re.sub(r'([\\d,.]+)','', tweet_text.lower()))\n",
    "\n",
    "    # Setting url value (whether the tweet contains http link) and filter http links\n",
    "    url=0\n",
    "    for token in tokens:\n",
    "        if token.startswith('http'):\n",
    "            url=1\n",
    "    tokens = [token for token in tokens if not token.startswith('http')]\n",
    "\n",
    "    ## Stemming\n",
    "    # porter = PorterStemmer()\n",
    "    # tokens = [porter.stem(token) for token in tokens]\n",
    "\n",
    "    # Filtering Stop words\n",
    "    # from nltk.corpus import stopwords\n",
    "    # stop_words = set(stopwords.words('english'))\n",
    "    # tokens = [token for token in tokens if not token in stop_words]\n",
    "\n",
    "    return tokens, url\n",
    "\n",
    "def getposcount(tokens):\n",
    "    postag = []\n",
    "    poscount = {}\n",
    "    poscount['Noun']=0\n",
    "    poscount['Verb']=0\n",
    "    poscount['Adjective'] = 0\n",
    "    poscount['Pronoun']=0\n",
    "    poscount['FirstPersonPronoun']=0\n",
    "    poscount['SecondPersonPronoun']=0\n",
    "    poscount['ThirdPersonPronoun']=0\n",
    "    poscount['Adverb']=0\n",
    "    poscount['Numeral']=0\n",
    "    poscount['Conjunction_inj']=0\n",
    "    poscount['Particle']=0\n",
    "    poscount['Determiner']=0\n",
    "    poscount['Modal']=0\n",
    "    poscount['Whs']=0\n",
    "    Nouns = {'NN','NNS','NNP','NNPS'}\n",
    "    Adverbs = {'RB','RBR','RBS'}\n",
    "    Whs = {'WDT','WP','WRB'} # Composition of wh-determiner(that,what), wh-pronoun(who), wh-adverb(how)\n",
    "    Verbs={'VB','VBP','VBZ','VBN','VBG','VBD','To'}\n",
    "    first_person_pronouns=['i','I','me','my','mine','we','us','our','ours'] #'i',\n",
    "    second_person_pronouns=['you','your','yours']\n",
    "    third_person_pronouns=['he','she','it','him','her','it','his','hers','its','they','them','their','theirs']\n",
    "\n",
    "    for word in tokens:\n",
    "        w_lower=word.lower()\n",
    "        if w_lower in first_person_pronouns:\n",
    "            poscount['FirstPersonPronoun']+=1\n",
    "        elif w_lower in second_person_pronouns:\n",
    "            poscount['SecondPersonPronoun']+=1\n",
    "        elif w_lower in third_person_pronouns:\n",
    "            poscount['ThirdPersonPronoun']+=1\n",
    "    \n",
    "    postag = nltk.pos_tag(tokens)\n",
    "    for g1 in postag:\n",
    "        if g1[1] in Nouns:\n",
    "            poscount['Noun'] += 1\n",
    "        elif g1[1] in Verbs:\n",
    "            poscount['Verb']+= 1\n",
    "        elif g1[1]=='ADJ'or g1[1]=='JJ':\n",
    "            poscount['Adjective']+=1\n",
    "        elif g1[1]=='PRP' or g1[1]=='PRON' or g1[1]=='PRP$':\n",
    "            poscount['Pronoun']+=1\n",
    "        elif g1[1] in Adverbs or g1[1]=='ADV':\n",
    "            poscount['Adverb']+=1\n",
    "        elif g1[1]=='CD':\n",
    "            poscount['Numeral']+=1\n",
    "        elif g1[1]=='CC' or g1[1]=='IN':\n",
    "            poscount['Conjunction_inj']+=1\n",
    "        elif g1[1]=='RP':\n",
    "            poscount['Particle']+=1\n",
    "        elif g1[1]=='MD':\n",
    "            poscount['Modal']+=1\n",
    "        elif g1[1]=='DT':\n",
    "            poscount['Determiner']+=1\n",
    "        elif g1[1] in Whs:\n",
    "            poscount['Whs']+=1\n",
    "    return poscount\n",
    "\n",
    "def contentlength(words):\n",
    "    wordcount = len(words)\n",
    "    return wordcount\n",
    "\n",
    "def extract_urls(entities_dicts):\n",
    "    if len(entities_dicts) < 1:\n",
    "        return 0,[],[]\n",
    "\n",
    "    urls = []\n",
    "    urls_expanded = []\n",
    "\n",
    "    key = 'url'\n",
    "    key2 = 'expanded_url'\n",
    "    # print(len(entities_dict))\n",
    "    for i in entities_dicts:\n",
    "        urls.append(i[key])\n",
    "        urls_expanded.append(i[key2])\n",
    "    return 1, urls, urls_expanded\n",
    "\n",
    "def flatten_tweets(tweets):\n",
    "    \"\"\" Flattens out tweet dictionaries so relevant JSON is in a top-level dictionary. \"\"\"\n",
    "    tweets_list = []\n",
    "    total_tokens_l = []\n",
    "\n",
    "    # Iterate through each tweet\n",
    "    for tweet_obj in tweets:\n",
    "        output_f = dict()\n",
    "\n",
    "        output_f['text']= tweet_obj['text']\n",
    "        \n",
    "        urls_dicts = tweet_obj['entities']['urls']\n",
    "        # print(urls_dicts)\n",
    "\n",
    "        output_f['hasURL'], output_f['urls'], output_f['urls_expanded'] = extract_urls(urls_dicts)\n",
    "        \n",
    "        # print(type(tweet_obj['user']))\n",
    "        # print(tweet_obj['user'].contains_key('entities'))\n",
    "        if ('url' in tweet_obj['user']):\n",
    "            output_f['hasUserURL'] = 1\n",
    "            output_f['user_url'] = tweet_obj['user']['url']\n",
    "        elif ('entities' in tweet_obj['user']):\n",
    "            # output_f['user_entity'] = tweet_obj['user']['entities']['url']['urls']\n",
    "            # print(tweet_obj['user']['entities']['url']['urls'])\n",
    "            # output_f['user_url'] = tweet_obj['user']['entities']['expanded_url']\n",
    "            output_f['hasUserURL'] , _ , output_f['user_url'] = extract_urls(tweet_obj['user']['entities']['url']['urls'])\n",
    "        else:\n",
    "            # output_f['user_entity'] = None\n",
    "            output_f['user_url'] = 0\n",
    "            output_f['hasUserURL'] = 0\n",
    "\n",
    "        output_f['text_token'], output_f['isNotOnlyText'] = tweets2tokens(tweet_obj['text'])\n",
    "        total_tokens_l.extend(output_f['text_token']) # append the tokens to list of total tokens\n",
    "\n",
    "        '''POS Tagging'''\n",
    "        pos_dict=getposcount(output_f['text_token'])\n",
    "        output_f.update(pos_dict)\n",
    "\n",
    "        output_f['char_count'] = len(output_f['text'])\n",
    "        output_f['word_count'] = len(output_f['text_token'])\n",
    "\n",
    "        output_f['has_question'] = \"?\" in output_f[\"text\"]\n",
    "        output_f['has_exclaim'] = \"!\" in output_f[\"text\"]\n",
    "        output_f['has_period'] = \".\" in output_f[\"text\"]\n",
    "    \n",
    "        ''' User info'''\n",
    "        # Store the user screen name in 'user-screen_name'\n",
    "        # output_f['user-screen_name'] = tweet_obj['user']['screen_name']\n",
    "        \n",
    "        # Store the user location\n",
    "        # output_f['user-location'] = tweet_obj['user']['location']\n",
    "\n",
    "        acc_created = datetime.strptime(tweet_obj['user']['created_at'], '%a %b %d %H:%M:%S %z %Y')\n",
    "        tweet_created = datetime.strptime(tweet_obj['created_at'], '%a %b %d %H:%M:%S %z %Y')\n",
    "        age = (tweet_created - acc_created)\n",
    "        # print(type(timedelta.total_seconds(age)))\n",
    "\n",
    "        output_f['capital_ratio']=(capitalratio(tweet_obj['text']))\n",
    "\n",
    "        # features=(capitalratio(data_list[0]['user']))\n",
    "        output_f['tweet_count'] = np.log10(tweet_obj['user']['statuses_count'])\n",
    "        output_f['listed_count'] = np.log10(tweet_obj['user']['listed_count'])\n",
    "        output_f['follow_ratio'] = np.log10(tweet_obj['user']['followers_count'])\n",
    "        output_f['age'] = int(timedelta.total_seconds(age)/86400)\n",
    "        output_f['verified'] = tweet_obj['user']['verified']\n",
    "\n",
    "        tweets_list.append(output_f)\n",
    "\n",
    "    unk_tokens_l = list(set(total_tokens_l))\n",
    "    print(\"Number of total tokens appeared: {}\\nNumber of unique tokens appeared: {}\\n\".format(len(total_tokens_l), len(unk_tokens_l))) # number of tokens and unique tokens\n",
    "\n",
    "    return tweets_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "class data_loader():\n",
    "\n",
    "    def getdata(self, root=True, reaction=False, split=True):\n",
    "        lists = []\n",
    "        if root == True:\n",
    "            charliehebdo_jsons = glob('../pheme-rnr-dataset/charliehebdo/**/source-tweet/*.json') \n",
    "            ferguson_jsons = glob('../pheme-rnr-dataset/ferguson/**/source-tweet/*.json')\n",
    "            germanwing_scrash_jsons = glob('../pheme-rnr-dataset/germanwings-crash/**/source-tweet/*.json')\n",
    "            ottawashooting_jsons = glob('../pheme-rnr-dataset/ottawashooting/**/source-tweet/*.json')\n",
    "            sydneysiege_jsons = glob('../pheme-rnr-dataset/sydneysiege/**/source-tweet/*.json')\n",
    "            print(len(charliehebdo_jsons),len(ferguson_jsons),len(germanwing_scrash_jsons),len(ottawashooting_jsons),len(sydneysiege_jsons))\n",
    "            lists.append([charliehebdo_jsons, ferguson_jsons, germanwing_scrash_jsons, ottawashooting_jsons, sydneysiege_jsons])\n",
    "        elif reaction == True:\n",
    "            charliehebdo_reaction = glob('../pheme-rnr-dataset/charliehebdo/**/reactions/*.json') \n",
    "            ferguson_reaction = glob('../pheme-rnr-dataset/ferguson/**/reactions/*.json')\n",
    "            germanwing_scrash_reaction = glob('../pheme-rnr-dataset/germanwings-crash/**/reactions/*.json')\n",
    "            ottawashooting_reaction = glob('../pheme-rnr-dataset/ottawashooting/**/reactions/*.json')\n",
    "            sydneysiege_reaction = glob('../pheme-rnr-dataset/sydneysiege/**/reactions/*.json')\n",
    "            lists.append([charliehebdo_reaction, ferguson_reaction, germanwing_scrash_reaction, ottawashooting_reaction, sydneysiege_reaction])\n",
    "            \n",
    "        return lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2079 1143 469 890 1221\n"
     ]
    }
   ],
   "source": [
    "jsonFiles = data_loader().getdata()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Importing JSON Files and grouping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2079 1143 469 890 1221\n"
     ]
    }
   ],
   "source": [
    "jsonFiles = data_loader().getdata()\n",
    "data = cross_val_jsons(jsonFiles[0], isTest = True)\n",
    "# data = cross_val_jsons(jsonFiles[0], isTest = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Data of Root tweets) Train: 3723 Test: 2079\n",
      "(Data of Root tweets) Train_y: 3723 Test_y: 2079\n"
     ]
    }
   ],
   "source": [
    "data_lists, isRumorLists = extract_data(data)\n",
    "\n",
    "train, test = data_lists[0], data_lists[1]\n",
    "df_train_y, df_test_y = isRumorLists[0],isRumorLists[1]\n",
    "print(\"(Data of Root tweets) Train: {} Test: {}\".format(len(train),len(test)))\n",
    "print(\"(Data of Root tweets) Train_y: {} Test_y: {}\".format(len(isRumorLists[0]),len(isRumorLists[1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>0</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>[{'contributors': None, 'truncated': False, 'text': 'Michael Brown was suspected of robbing store in #Ferguson before being shot, say police http://t.co/KUZQGasFgA http://t.co/TjuEVB8z5m', 'in_reply_to_status_id': None, 'id': 500311153583853570, 'favorite_count': 88, 'source': '&lt;a href=\"http://www.socialflow.com\" rel=\"nofollow\"&gt;SocialFlow&lt;/a&gt;', 'retweeted': False, 'coordinates': None, 'entitie...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>[{'contributors': None, 'truncated': False, 'text': 'BREAKING: Armed man takes hostage in kosher grocery east of Paris http://t.co/PBs3sMwhLt', 'in_reply_to_status_id': None, 'id': 553529101659566080, 'favorite_count': 14, 'source': '&lt;a href=\"https://about.twitter.com/products/tweetdeck\" rel=\"nofollow\"&gt;TweetDeck&lt;/a&gt;', 'retweeted': False, 'coordinates': None, 'entities': {'user_mentions': [], '...</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                                                                 0\n",
       "0  [{'contributors': None, 'truncated': False, 'text': 'Michael Brown was suspected of robbing store in #Ferguson before being shot, say police http://t.co/KUZQGasFgA http://t.co/TjuEVB8z5m', 'in_reply_to_status_id': None, 'id': 500311153583853570, 'favorite_count': 88, 'source': '<a href=\"http://www.socialflow.com\" rel=\"nofollow\">SocialFlow</a>', 'retweeted': False, 'coordinates': None, 'entitie...\n",
       "1  [{'contributors': None, 'truncated': False, 'text': 'BREAKING: Armed man takes hostage in kosher grocery east of Paris http://t.co/PBs3sMwhLt', 'in_reply_to_status_id': None, 'id': 553529101659566080, 'favorite_count': 14, 'source': '<a href=\"https://about.twitter.com/products/tweetdeck\" rel=\"nofollow\">TweetDeck</a>', 'retweeted': False, 'coordinates': None, 'entities': {'user_mentions': [], '..."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(np.array(data_lists))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_lists[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Data of Root tweets) Data: 5802\n",
      "(Data of Root tweets) Label: 5802\n"
     ]
    }
   ],
   "source": [
    "jsonFiles = data_loader().getdata()\n",
    "data = cross_val_jsons(jsonFiles[0], isTest = False)\n",
    "data_lists, isRumorLists = extract_data(data)\n",
    "\n",
    "data = data_lists[0]\n",
    "data_y = isRumorLists[0]\n",
    "print(\"(Data of Root tweets) Data: {}\".format(len(data)))\n",
    "print(\"(Data of Root tweets) Label: {}\".format(len(data_y)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Flatten and extract basic features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of total tokens appeared: 69679\n",
      "Number of unique tokens appeared: 6761\n",
      "\n",
      "Number of total tokens appeared: 18496\n",
      "Number of unique tokens appeared: 3149\n",
      "\n",
      "Number of total tokens appeared: 88175\n",
      "Number of unique tokens appeared: 8154\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_train = pd.DataFrame(flatten_tweets(train))\n",
    "df_test = pd.DataFrame(flatten_tweets(test))\n",
    "df_data = pd.DataFrame(flatten_tweets(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>text</th>\n      <th>hasURL</th>\n      <th>urls</th>\n      <th>urls_expanded</th>\n      <th>hasUserURL</th>\n      <th>user_url</th>\n      <th>text_token</th>\n      <th>isNotOnlyText</th>\n      <th>Noun</th>\n      <th>Verb</th>\n      <th>Adjective</th>\n      <th>Pronoun</th>\n      <th>FirstPersonPronoun</th>\n      <th>SecondPersonPronoun</th>\n      <th>ThirdPersonPronoun</th>\n      <th>Adverb</th>\n      <th>Numeral</th>\n      <th>Conjunction_inj</th>\n      <th>Particle</th>\n      <th>Determiner</th>\n      <th>Modal</th>\n      <th>Whs</th>\n      <th>char_count</th>\n      <th>word_count</th>\n      <th>has_question</th>\n      <th>has_exclaim</th>\n      <th>has_period</th>\n      <th>capital_ratio</th>\n      <th>tweet_count</th>\n      <th>listed_count</th>\n      <th>follow_ratio</th>\n      <th>age</th>\n      <th>verified</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>BREAKING: Armed man takes hostage in kosher grocery east of Paris http://t.co/PBs3sMwhLt</td>\n      <td>1</td>\n      <td>[http://t.co/PBs3sMwhLt]</td>\n      <td>[http://htz.li/1lI]</td>\n      <td>1</td>\n      <td>http://www.haaretz.com</td>\n      <td>[breaking, armed, man, takes, hostage, in, kosher, grocery, east, of, paris]</td>\n      <td>1</td>\n      <td>6</td>\n      <td>3</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>88</td>\n      <td>11</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>0.159091</td>\n      <td>4.803286</td>\n      <td>3.855943</td>\n      <td>5.287349</td>\n      <td>2126</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>#CharlieHebdo killers dead, confirmed by gendarmerie.</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>1</td>\n      <td>http://www.agnespoirier.org</td>\n      <td>[charliehebdo, killers, dead, confirmed, by, gendarmerie]</td>\n      <td>0</td>\n      <td>2</td>\n      <td>1</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>53</td>\n      <td>6</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>0.037736</td>\n      <td>3.031812</td>\n      <td>2.146128</td>\n      <td>3.672929</td>\n      <td>1050</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Top French cartoonists Charb, Cabu, Wolinski, Tignous confirmed among dead in #Paris #CharlieHebdo attack. Editor is critically wounded.</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>1</td>\n      <td>http://t.co/EWb7m4orG8</td>\n      <td>[top, french, cartoonists, charb, cabu, wolinski, tignous, confirmed, among, dead, in, paris, charliehebdo, attack, editor, is, critically, wounded]</td>\n      <td>0</td>\n      <td>3</td>\n      <td>4</td>\n      <td>8</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>136</td>\n      <td>18</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>0.073529</td>\n      <td>3.856245</td>\n      <td>2.879669</td>\n      <td>4.309651</td>\n      <td>2030</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Police have surrounded the area where the #CharlieHebdo attack suspects are believed to be: http://t.co/3tGXEIX4F2\\nhttps://t.co/aBSezf2QWS</td>\n      <td>1</td>\n      <td>[http://t.co/3tGXEIX4F2, https://t.co/aBSezf2QWS]</td>\n      <td>[http://cnn.it/1xYDHvp, https://amp.twimg.com/v/7d7ecf3e-0965-41ca-afa6-81397b4854db]</td>\n      <td>1</td>\n      <td>http://www.cnn.com</td>\n      <td>[police, have, surrounded, the, area, where, the, charliehebdo, attack, suspects, are, believed, to, be]</td>\n      <td>1</td>\n      <td>5</td>\n      <td>5</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n      <td>1</td>\n      <td>138</td>\n      <td>14</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>0.101449</td>\n      <td>4.735814</td>\n      <td>5.009820</td>\n      <td>7.187664</td>\n      <td>2891</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>PHOTO: Armed gunmen face police officers near #CharlieHebdo HQ in Paris http://t.co/3Jsosc7yl3 http://t.co/iOpVNO6Iq0</td>\n      <td>1</td>\n      <td>[http://t.co/3Jsosc7yl3]</td>\n      <td>[http://on.rt.com/k5ivya]</td>\n      <td>1</td>\n      <td>http://t.co/bDDyvy9DmR</td>\n      <td>[photo, armed, gunmen, face, police, officers, near, charliehebdo, hq, in, paris]</td>\n      <td>1</td>\n      <td>7</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>117</td>\n      <td>11</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>0.145299</td>\n      <td>5.021181</td>\n      <td>4.132996</td>\n      <td>5.925434</td>\n      <td>1975</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>5797</th>\n      <td>'I'll ride with you' http://t.co/llZnuCAzg5 Australia unites during #SydneySiege http://t.co/WIU22VPgkz</td>\n      <td>1</td>\n      <td>[http://t.co/llZnuCAzg5]</td>\n      <td>[http://bbc.in/1DArzWS]</td>\n      <td>1</td>\n      <td>http://bbc.com/trending</td>\n      <td>[ill, ride, with, you, australia, unites, during, sydneysiege]</td>\n      <td>1</td>\n      <td>2</td>\n      <td>2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>103</td>\n      <td>8</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>0.116505</td>\n      <td>3.586024</td>\n      <td>2.876795</td>\n      <td>4.716070</td>\n      <td>531</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>5798</th>\n      <td>Canada's thoughts and prayers are with our Australian friends. #MartinPlace #SydneySiege</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>1</td>\n      <td>http://t.co/L2SH1QDkAY</td>\n      <td>[canadas, thoughts, and, prayers, are, with, our, australian, friends, martinplace, sydneysiege]</td>\n      <td>0</td>\n      <td>5</td>\n      <td>2</td>\n      <td>1</td>\n      <td>1</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>88</td>\n      <td>11</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>0.068182</td>\n      <td>3.466423</td>\n      <td>3.893429</td>\n      <td>5.800086</td>\n      <td>2713</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>5799</th>\n      <td>Every non-muslim in the world must watch this video https://t.co/sZdyhISoVh &amp;amp; show it every other non-muslim! #sydneysiege</td>\n      <td>1</td>\n      <td>[https://t.co/sZdyhISoVh]</td>\n      <td>[https://www.youtube.com/watch?v=d8c38_46W5c]</td>\n      <td>1</td>\n      <td>http://t.co/uTUmOx49Zj</td>\n      <td>[every, nonmuslim, in, the, world, must, watch, this, video, amp, show, it, every, other, nonmuslim, sydneysiege]</td>\n      <td>1</td>\n      <td>4</td>\n      <td>3</td>\n      <td>2</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>4</td>\n      <td>1</td>\n      <td>0</td>\n      <td>126</td>\n      <td>16</td>\n      <td>False</td>\n      <td>True</td>\n      <td>True</td>\n      <td>0.039683</td>\n      <td>3.681151</td>\n      <td>1.924279</td>\n      <td>4.557531</td>\n      <td>549</td>\n      <td>False</td>\n    </tr>\n    <tr>\n      <th>5800</th>\n      <td>Suspect in Sydney cafe siege identified as Man Haron Monis, an Iranian granted asylum in Australia http://t.co/6Lrl9DEMXA</td>\n      <td>1</td>\n      <td>[http://t.co/6Lrl9DEMXA]</td>\n      <td>[http://bbc.in/1znVJHB]</td>\n      <td>1</td>\n      <td>http://www.bbc.co.uk/news</td>\n      <td>[suspect, in, sydney, cafe, siege, identified, as, man, haron, monis, an, iranian, granted, asylum, in, australia]</td>\n      <td>1</td>\n      <td>7</td>\n      <td>3</td>\n      <td>2</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>3</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>121</td>\n      <td>16</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>0.107438</td>\n      <td>4.349879</td>\n      <td>5.007671</td>\n      <td>7.097693</td>\n      <td>2793</td>\n      <td>True</td>\n    </tr>\n    <tr>\n      <th>5801</th>\n      <td>Australians respond to racism by telling #Muslim community #illridewithyou. #sydneysiege #MartinPlace</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>1</td>\n      <td>http://t.co/fXo4ZyqsDh</td>\n      <td>[australians, respond, to, racism, by, telling, muslim, community, illridewithyou, sydneysiege, martinplace]</td>\n      <td>0</td>\n      <td>5</td>\n      <td>4</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>101</td>\n      <td>11</td>\n      <td>False</td>\n      <td>False</td>\n      <td>True</td>\n      <td>0.039604</td>\n      <td>4.245562</td>\n      <td>2.738781</td>\n      <td>4.164650</td>\n      <td>2117</td>\n      <td>False</td>\n    </tr>\n  </tbody>\n</table>\n<p>5802 rows × 33 columns</p>\n</div>",
      "text/plain": [
       "                                                                                                                                             text  \\\n",
       "0                                                        BREAKING: Armed man takes hostage in kosher grocery east of Paris http://t.co/PBs3sMwhLt   \n",
       "1                                                                                           #CharlieHebdo killers dead, confirmed by gendarmerie.   \n",
       "2        Top French cartoonists Charb, Cabu, Wolinski, Tignous confirmed among dead in #Paris #CharlieHebdo attack. Editor is critically wounded.   \n",
       "3     Police have surrounded the area where the #CharlieHebdo attack suspects are believed to be: http://t.co/3tGXEIX4F2\\nhttps://t.co/aBSezf2QWS   \n",
       "4                           PHOTO: Armed gunmen face police officers near #CharlieHebdo HQ in Paris http://t.co/3Jsosc7yl3 http://t.co/iOpVNO6Iq0   \n",
       "...                                                                                                                                           ...   \n",
       "5797                                      'I'll ride with you' http://t.co/llZnuCAzg5 Australia unites during #SydneySiege http://t.co/WIU22VPgkz   \n",
       "5798                                                     Canada's thoughts and prayers are with our Australian friends. #MartinPlace #SydneySiege   \n",
       "5799               Every non-muslim in the world must watch this video https://t.co/sZdyhISoVh &amp; show it every other non-muslim! #sydneysiege   \n",
       "5800                    Suspect in Sydney cafe siege identified as Man Haron Monis, an Iranian granted asylum in Australia http://t.co/6Lrl9DEMXA   \n",
       "5801                                        Australians respond to racism by telling #Muslim community #illridewithyou. #sydneysiege #MartinPlace   \n",
       "\n",
       "      hasURL                                               urls  \\\n",
       "0          1                           [http://t.co/PBs3sMwhLt]   \n",
       "1          0                                                 []   \n",
       "2          0                                                 []   \n",
       "3          1  [http://t.co/3tGXEIX4F2, https://t.co/aBSezf2QWS]   \n",
       "4          1                           [http://t.co/3Jsosc7yl3]   \n",
       "...      ...                                                ...   \n",
       "5797       1                           [http://t.co/llZnuCAzg5]   \n",
       "5798       0                                                 []   \n",
       "5799       1                          [https://t.co/sZdyhISoVh]   \n",
       "5800       1                           [http://t.co/6Lrl9DEMXA]   \n",
       "5801       0                                                 []   \n",
       "\n",
       "                                                                              urls_expanded  \\\n",
       "0                                                                       [http://htz.li/1lI]   \n",
       "1                                                                                        []   \n",
       "2                                                                                        []   \n",
       "3     [http://cnn.it/1xYDHvp, https://amp.twimg.com/v/7d7ecf3e-0965-41ca-afa6-81397b4854db]   \n",
       "4                                                                 [http://on.rt.com/k5ivya]   \n",
       "...                                                                                     ...   \n",
       "5797                                                                [http://bbc.in/1DArzWS]   \n",
       "5798                                                                                     []   \n",
       "5799                                          [https://www.youtube.com/watch?v=d8c38_46W5c]   \n",
       "5800                                                                [http://bbc.in/1znVJHB]   \n",
       "5801                                                                                     []   \n",
       "\n",
       "      hasUserURL                     user_url  \\\n",
       "0              1       http://www.haaretz.com   \n",
       "1              1  http://www.agnespoirier.org   \n",
       "2              1       http://t.co/EWb7m4orG8   \n",
       "3              1           http://www.cnn.com   \n",
       "4              1       http://t.co/bDDyvy9DmR   \n",
       "...          ...                          ...   \n",
       "5797           1      http://bbc.com/trending   \n",
       "5798           1       http://t.co/L2SH1QDkAY   \n",
       "5799           1       http://t.co/uTUmOx49Zj   \n",
       "5800           1    http://www.bbc.co.uk/news   \n",
       "5801           1       http://t.co/fXo4ZyqsDh   \n",
       "\n",
       "                                                                                                                                                text_token  \\\n",
       "0                                                                             [breaking, armed, man, takes, hostage, in, kosher, grocery, east, of, paris]   \n",
       "1                                                                                                [charliehebdo, killers, dead, confirmed, by, gendarmerie]   \n",
       "2     [top, french, cartoonists, charb, cabu, wolinski, tignous, confirmed, among, dead, in, paris, charliehebdo, attack, editor, is, critically, wounded]   \n",
       "3                                                 [police, have, surrounded, the, area, where, the, charliehebdo, attack, suspects, are, believed, to, be]   \n",
       "4                                                                        [photo, armed, gunmen, face, police, officers, near, charliehebdo, hq, in, paris]   \n",
       "...                                                                                                                                                    ...   \n",
       "5797                                                                                        [ill, ride, with, you, australia, unites, during, sydneysiege]   \n",
       "5798                                                      [canadas, thoughts, and, prayers, are, with, our, australian, friends, martinplace, sydneysiege]   \n",
       "5799                                     [every, nonmuslim, in, the, world, must, watch, this, video, amp, show, it, every, other, nonmuslim, sydneysiege]   \n",
       "5800                                    [suspect, in, sydney, cafe, siege, identified, as, man, haron, monis, an, iranian, granted, asylum, in, australia]   \n",
       "5801                                          [australians, respond, to, racism, by, telling, muslim, community, illridewithyou, sydneysiege, martinplace]   \n",
       "\n",
       "      isNotOnlyText  Noun  Verb  Adjective  Pronoun  FirstPersonPronoun  \\\n",
       "0                 1     6     3          0        0                   0   \n",
       "1                 0     2     1          2        0                   0   \n",
       "2                 0     3     4          8        0                   0   \n",
       "3                 1     5     5          0        0                   0   \n",
       "4                 1     7     2          0        0                   0   \n",
       "...             ...   ...   ...        ...      ...                 ...   \n",
       "5797              1     2     2          1        1                   0   \n",
       "5798              0     5     2          1        1                   1   \n",
       "5799              1     4     3          2        1                   0   \n",
       "5800              1     7     3          2        0                   0   \n",
       "5801              0     5     4          0        0                   0   \n",
       "\n",
       "      SecondPersonPronoun  ThirdPersonPronoun  Adverb  Numeral  \\\n",
       "0                       0                   0       0        0   \n",
       "1                       0                   0       0        0   \n",
       "2                       0                   0       1        0   \n",
       "3                       0                   0       0        0   \n",
       "4                       0                   0       0        0   \n",
       "...                   ...                 ...     ...      ...   \n",
       "5797                    1                   0       0        0   \n",
       "5798                    0                   0       0        0   \n",
       "5799                    0                   1       0        0   \n",
       "5800                    0                   0       0        0   \n",
       "5801                    0                   0       0        0   \n",
       "\n",
       "      Conjunction_inj  Particle  Determiner  Modal  Whs  char_count  \\\n",
       "0                   2         0           0      0    0          88   \n",
       "1                   1         0           0      0    0          53   \n",
       "2                   2         0           0      0    0         136   \n",
       "3                   0         0           2      0    1         138   \n",
       "4                   2         0           0      0    0         117   \n",
       "...               ...       ...         ...    ...  ...         ...   \n",
       "5797                2         0           0      0    0         103   \n",
       "5798                2         0           0      0    0          88   \n",
       "5799                1         0           4      1    0         126   \n",
       "5800                3         0           1      0    0         121   \n",
       "5801                1         0           0      0    0         101   \n",
       "\n",
       "      word_count  has_question  has_exclaim  has_period  capital_ratio  \\\n",
       "0             11         False        False        True       0.159091   \n",
       "1              6         False        False        True       0.037736   \n",
       "2             18         False        False        True       0.073529   \n",
       "3             14         False        False        True       0.101449   \n",
       "4             11         False        False        True       0.145299   \n",
       "...          ...           ...          ...         ...            ...   \n",
       "5797           8         False        False        True       0.116505   \n",
       "5798          11         False        False        True       0.068182   \n",
       "5799          16         False         True        True       0.039683   \n",
       "5800          16         False        False        True       0.107438   \n",
       "5801          11         False        False        True       0.039604   \n",
       "\n",
       "      tweet_count  listed_count  follow_ratio   age  verified  \n",
       "0        4.803286      3.855943      5.287349  2126      True  \n",
       "1        3.031812      2.146128      3.672929  1050     False  \n",
       "2        3.856245      2.879669      4.309651  2030     False  \n",
       "3        4.735814      5.009820      7.187664  2891      True  \n",
       "4        5.021181      4.132996      5.925434  1975      True  \n",
       "...           ...           ...           ...   ...       ...  \n",
       "5797     3.586024      2.876795      4.716070   531      True  \n",
       "5798     3.466423      3.893429      5.800086  2713      True  \n",
       "5799     3.681151      1.924279      4.557531   549     False  \n",
       "5800     4.349879      5.007671      7.097693  2793      True  \n",
       "5801     4.245562      2.738781      4.164650  2117     False  \n",
       "\n",
       "[5802 rows x 33 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train[['has_question', 'has_exclaim', 'has_period','verified']] = df_train[['has_question', 'has_exclaim', 'has_period','verified']].astype(int)\n",
    "df_test[['has_question', 'has_exclaim', 'has_period','verified']] = df_test[['has_question', 'has_exclaim', 'has_period','verified']].astype(int)\n",
    "df_data[['has_question', 'has_exclaim', 'has_period','verified']] = df_data[['has_question', 'has_exclaim', 'has_period','verified']].astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Handling inf and NaN value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(np.any(np.isnan(X)))\n",
    "# print(np.all(np.isfinite(X)))\n",
    "# print(X['listed_count'].mean())\n",
    "# print(pd.DataFrame(X['listed_count'].replace([np.inf, -np.inf], np.nan)).mean())\n",
    "\n",
    "# listed_nan = X[['listed_count','tweet_count','follow_ratio','age']].loc[X['listed_count']<0.2]\n",
    "# listed_nan = listed_nan.replace([np.inf, -np.inf], np.nan)\n",
    "# np.isnan(listed_nan).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.2022264175363997\n",
      "Before fill: Does the dataset contain NaN value? True\n",
      "After fill: Does the dataset contain NaN value? False\n",
      "2.6928269219616747\n",
      "Before fill: Does the dataset contain NaN value? True\n",
      "After fill: Does the dataset contain NaN value? False\n",
      "3.1020948474234245\n",
      "Before fill: Does the dataset contain NaN value? True\n",
      "After fill: Does the dataset contain NaN value? False\n"
     ]
    }
   ],
   "source": [
    "for dataset in [df_train, df_test, df_data]:\n",
    "    dataset['listed_count'].replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    print(dataset['listed_count'].mean())\n",
    "    print(\"Before fill: Does the dataset contain NaN value? {}\".format(np.any(np.isnan(dataset['listed_count']))))\n",
    "    dataset['listed_count'].fillna(0,inplace=True)\n",
    "    print(\"After fill: Does the dataset contain NaN value? {}\".format(np.any(np.isnan(dataset['listed_count']))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_data = pd.concat([df_data,data_y],axis=1)\n",
    "df_data.to_csv('./data/data_notembeded.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reaction files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reactionFiles = data_loader().getdata(reaction=True,root=False)\n",
    "data_rt = cross_val_jsons(reactionFiles[0], isTest = False)\n",
    "data_lists, isRumorLists = extract_data(data_rt)\n",
    "\n",
    "data_rt = data_lists[0]\n",
    "data_rt_y = isRumorLists[0]\n",
    "print(\"(Data of Root tweets) Data: {}\".format(len(data_rt)))\n",
    "print(\"(Data of Root tweets) Label: {}\".format(len(data_rt_y)))\n",
    "\n",
    "df_data_rt = pd.DataFrame(flatten_tweets(data_rt))\n",
    "\n",
    "df_data_rt[['has_question', 'has_exclaim', 'has_period','verified']] = df_data_rt[['has_question', 'has_exclaim', 'has_period','verified']].astype(int)\n",
    "\n",
    "for dataset in [df_data_rt]:\n",
    "    dataset['listed_count'].replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    print(dataset['listed_count'].mean())\n",
    "    print(\"Before fill: Does the dataset contain NaN value? {}\".format(np.any(np.isnan(dataset['listed_count']))))\n",
    "    dataset['listed_count'].fillna(0,inplace=True)\n",
    "    print(\"After fill: Does the dataset contain NaN value? {}\".format(np.any(np.isnan(dataset['listed_count']))))\n",
    "print(data_rt.shape, data_rt_y.shape)\n",
    "df_data_rt.to_csv('./data/data_rt_notembeded.csv', index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec_ = w2v.Word2Vec(\n",
    "    df_train['text_token'],\n",
    "    sg = 1, \n",
    "    seed = 1,\n",
    "    workers = 8,\n",
    "    size = 300,\n",
    "    min_count = 5,\n",
    "    window = 10,\n",
    "    sample = 1e-3\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(246411, 372950)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word2vec_.train(df_train['text_token'], total_examples = word2vec_.corpus_count, epochs = word2vec_.iter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "# word2vec_.wv.vectors.shape # vocab size / window size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec_.save('w2v_model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Word Vector model and Vectorize the sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec_ = w2v.Word2Vec.load('w2v_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'word2vec_' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-4efd874c6cd3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mword_vectors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mword2vec_\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwv\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mvocabs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mword_vectors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocab\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'word2vec_' is not defined"
     ]
    }
   ],
   "source": [
    "word_vectors = word2vec_.wv\n",
    "vocabs = word_vectors.vocab.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_vectors = word_vectors.vectors # here you load vectors for each word in your model\n",
    "w2v_indices = {word: word_vectors.vocab[word].index for word in word_vectors.vocab} # here you load indices - with whom you can find an index of the particular word in your model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# word2vec_.most_similar('liniers')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize(line): \n",
    "    words = []\n",
    "    for word in line: # line - iterable, for example list of tokens \n",
    "        try:\n",
    "            w2v_idx = w2v_indices[word]\n",
    "        except KeyError: # if you does not have a vector for this word in your w2v model, continue \n",
    "            continue\n",
    "        words.append(list(w2v_vectors[w2v_idx]))\n",
    "        if not word:\n",
    "            words.append(None)\n",
    "\n",
    "        if len(line) > len(words):\n",
    "            continue\n",
    "    return np.asarray(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(\"Tweet 1: \", df['text'][1])\n",
    "# print(\"Indice of '{}': {}\".format(df['text_token'][1][0], w2v_indices[df['text_token'][1][0]]))\n",
    "# print(\"Indice of '{}': {}\".format(df['text_token'][1][0], w2v_vectors[w2v_indices[df['text_token'][1][0]]]))\n",
    "# print(\"Indice of '{}': {}\".format(df['text_token'][1][1], w2v_indices[df['text_token'][1][1]]))\n",
    "# print(\"Indice of '{}': {}\".format(df['text_token'][1][1], w2v_vectors[w2v_indices[df['text_token'][1][1]]]))\n",
    "# print(\"\\nVector of the first headline:\\n\", vectorize(df['text_token'][1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Average of Vectors & Previous features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "df_train['text_token_vec'] = copy.deepcopy(df_train['text_token'])\n",
    "\n",
    "for index, sentence in enumerate(df_train['text_token_vec']):\n",
    "    df_train['text_token_vec'][index] = vectorize(sentence).mean(axis=0)\n",
    "\n",
    "pd.DataFrame(df_train['text_token_vec'].values.tolist()).shape\n",
    "df_train[['text_token','text_token_vec']].head()\n",
    "df_train_avg = pd.DataFrame(df_train['text_token_vec'].values.tolist()).add_prefix('token_avg') #.join(df)\n",
    "df_train_avg = df_train.join(df_train_avg).drop('text_token_vec',axis=1)\n",
    "df_train.drop(['text_token_vec'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>text</th>\n      <th>hasURL</th>\n      <th>urls</th>\n      <th>urls_expanded</th>\n      <th>hasUserURL</th>\n      <th>user_url</th>\n      <th>text_token</th>\n      <th>isNotOnlyText</th>\n      <th>Noun</th>\n      <th>Verb</th>\n      <th>...</th>\n      <th>token_avg290</th>\n      <th>token_avg291</th>\n      <th>token_avg292</th>\n      <th>token_avg293</th>\n      <th>token_avg294</th>\n      <th>token_avg295</th>\n      <th>token_avg296</th>\n      <th>token_avg297</th>\n      <th>token_avg298</th>\n      <th>token_avg299</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>BREAKING: Armed man takes hostage in kosher gr...</td>\n      <td>1</td>\n      <td>[http://t.co/PBs3sMwhLt]</td>\n      <td>[http://htz.li/1lI]</td>\n      <td>1</td>\n      <td>http://www.haaretz.com</td>\n      <td>[breaking, armed, man, takes, hostage, in, kos...</td>\n      <td>1</td>\n      <td>6</td>\n      <td>3</td>\n      <td>...</td>\n      <td>0.062943</td>\n      <td>0.132718</td>\n      <td>0.133037</td>\n      <td>-0.026727</td>\n      <td>0.031101</td>\n      <td>-0.130553</td>\n      <td>-0.055182</td>\n      <td>0.257094</td>\n      <td>0.000888</td>\n      <td>-0.071521</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>#CharlieHebdo killers dead, confirmed by genda...</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>1</td>\n      <td>http://www.agnespoirier.org</td>\n      <td>[charliehebdo, killers, dead, confirmed, by, g...</td>\n      <td>0</td>\n      <td>2</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0.043733</td>\n      <td>0.111315</td>\n      <td>-0.040369</td>\n      <td>-0.014255</td>\n      <td>-0.099877</td>\n      <td>-0.110036</td>\n      <td>0.002233</td>\n      <td>0.145399</td>\n      <td>-0.059819</td>\n      <td>-0.187999</td>\n    </tr>\n  </tbody>\n</table>\n<p>2 rows × 333 columns</p>\n</div>",
      "text/plain": [
       "                                                text  hasURL  \\\n",
       "0  BREAKING: Armed man takes hostage in kosher gr...       1   \n",
       "1  #CharlieHebdo killers dead, confirmed by genda...       0   \n",
       "\n",
       "                       urls        urls_expanded  hasUserURL  \\\n",
       "0  [http://t.co/PBs3sMwhLt]  [http://htz.li/1lI]           1   \n",
       "1                        []                   []           1   \n",
       "\n",
       "                      user_url  \\\n",
       "0       http://www.haaretz.com   \n",
       "1  http://www.agnespoirier.org   \n",
       "\n",
       "                                          text_token  isNotOnlyText  Noun  \\\n",
       "0  [breaking, armed, man, takes, hostage, in, kos...              1     6   \n",
       "1  [charliehebdo, killers, dead, confirmed, by, g...              0     2   \n",
       "\n",
       "   Verb  ...  token_avg290  token_avg291  token_avg292  token_avg293  \\\n",
       "0     3  ...      0.062943      0.132718      0.133037     -0.026727   \n",
       "1     1  ...      0.043733      0.111315     -0.040369     -0.014255   \n",
       "\n",
       "   token_avg294  token_avg295  token_avg296  token_avg297  token_avg298  \\\n",
       "0      0.031101     -0.130553     -0.055182      0.257094      0.000888   \n",
       "1     -0.099877     -0.110036      0.002233      0.145399     -0.059819   \n",
       "\n",
       "   token_avg299  \n",
       "0     -0.071521  \n",
       "1     -0.187999  \n",
       "\n",
       "[2 rows x 333 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train_avg.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Same work on X test "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "df_test['text_token_vec'] = copy.deepcopy(df_test['text_token'])\n",
    "\n",
    "for index, sentence in enumerate(df_test['text_token_vec']):\n",
    "    df_test['text_token_vec'][index] = vectorize(sentence).mean(axis=0)\n",
    "\n",
    "# df_test[['text_token','text_token_vec']].head()\n",
    "\n",
    "df_test_avg = pd.DataFrame(df_test['text_token_vec'].values.tolist()).add_prefix('token_avg')\n",
    "\n",
    "df_test_avg = df_test.join(df_test_avg).drop('text_token_vec',axis=1)\n",
    "df_test.drop('text_token_vec',axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>token_avg0</th>\n      <th>token_avg1</th>\n      <th>token_avg2</th>\n      <th>token_avg3</th>\n      <th>token_avg4</th>\n      <th>token_avg5</th>\n      <th>token_avg6</th>\n      <th>token_avg7</th>\n      <th>token_avg8</th>\n      <th>token_avg9</th>\n      <th>...</th>\n      <th>token_avg290</th>\n      <th>token_avg291</th>\n      <th>token_avg292</th>\n      <th>token_avg293</th>\n      <th>token_avg294</th>\n      <th>token_avg295</th>\n      <th>token_avg296</th>\n      <th>token_avg297</th>\n      <th>token_avg298</th>\n      <th>token_avg299</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.154436</td>\n      <td>0.042025</td>\n      <td>0.106603</td>\n      <td>-0.018409</td>\n      <td>0.055064</td>\n      <td>-0.110473</td>\n      <td>0.039882</td>\n      <td>0.002460</td>\n      <td>0.129744</td>\n      <td>0.058930</td>\n      <td>...</td>\n      <td>0.054728</td>\n      <td>0.009576</td>\n      <td>0.050590</td>\n      <td>-0.022001</td>\n      <td>-0.033597</td>\n      <td>-0.091051</td>\n      <td>-0.020110</td>\n      <td>0.163158</td>\n      <td>-0.010016</td>\n      <td>-0.018631</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.146316</td>\n      <td>-0.049514</td>\n      <td>0.130602</td>\n      <td>-0.045639</td>\n      <td>0.176577</td>\n      <td>-0.024466</td>\n      <td>0.055755</td>\n      <td>0.086503</td>\n      <td>0.113821</td>\n      <td>0.158433</td>\n      <td>...</td>\n      <td>0.052342</td>\n      <td>0.076558</td>\n      <td>0.056597</td>\n      <td>-0.020453</td>\n      <td>-0.081126</td>\n      <td>-0.084036</td>\n      <td>-0.028039</td>\n      <td>0.155239</td>\n      <td>-0.053626</td>\n      <td>-0.072127</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.244598</td>\n      <td>0.058895</td>\n      <td>0.154219</td>\n      <td>0.048585</td>\n      <td>0.136651</td>\n      <td>-0.036705</td>\n      <td>0.037018</td>\n      <td>0.064442</td>\n      <td>0.118488</td>\n      <td>0.129758</td>\n      <td>...</td>\n      <td>0.121074</td>\n      <td>0.190364</td>\n      <td>0.090983</td>\n      <td>-0.070175</td>\n      <td>-0.104205</td>\n      <td>-0.147921</td>\n      <td>-0.102687</td>\n      <td>0.291595</td>\n      <td>-0.087664</td>\n      <td>-0.141094</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.188610</td>\n      <td>-0.048544</td>\n      <td>0.041357</td>\n      <td>0.004833</td>\n      <td>0.138942</td>\n      <td>-0.055199</td>\n      <td>0.036372</td>\n      <td>0.059581</td>\n      <td>0.126967</td>\n      <td>0.102629</td>\n      <td>...</td>\n      <td>0.012302</td>\n      <td>0.096600</td>\n      <td>0.010848</td>\n      <td>-0.010891</td>\n      <td>-0.035067</td>\n      <td>-0.102598</td>\n      <td>-0.024998</td>\n      <td>0.137281</td>\n      <td>-0.099979</td>\n      <td>-0.037174</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.156274</td>\n      <td>-0.046768</td>\n      <td>0.135716</td>\n      <td>-0.028251</td>\n      <td>0.143405</td>\n      <td>-0.055834</td>\n      <td>0.020583</td>\n      <td>0.052493</td>\n      <td>0.124096</td>\n      <td>0.153234</td>\n      <td>...</td>\n      <td>0.001162</td>\n      <td>0.037805</td>\n      <td>0.047072</td>\n      <td>-0.028495</td>\n      <td>-0.064918</td>\n      <td>-0.107768</td>\n      <td>-0.049309</td>\n      <td>0.141919</td>\n      <td>-0.051378</td>\n      <td>-0.076649</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>885</th>\n      <td>0.189939</td>\n      <td>-0.091825</td>\n      <td>0.069886</td>\n      <td>-0.041547</td>\n      <td>0.160320</td>\n      <td>-0.005779</td>\n      <td>0.064590</td>\n      <td>0.092563</td>\n      <td>0.062800</td>\n      <td>0.122943</td>\n      <td>...</td>\n      <td>0.088572</td>\n      <td>0.095731</td>\n      <td>0.077700</td>\n      <td>-0.037176</td>\n      <td>-0.184658</td>\n      <td>-0.210214</td>\n      <td>-0.025299</td>\n      <td>0.182387</td>\n      <td>-0.119329</td>\n      <td>-0.084234</td>\n    </tr>\n    <tr>\n      <th>886</th>\n      <td>-0.046209</td>\n      <td>-0.078544</td>\n      <td>0.176929</td>\n      <td>-0.086290</td>\n      <td>0.044155</td>\n      <td>-0.112407</td>\n      <td>-0.064324</td>\n      <td>-0.039424</td>\n      <td>0.138600</td>\n      <td>0.129153</td>\n      <td>...</td>\n      <td>0.013887</td>\n      <td>-0.049760</td>\n      <td>0.096969</td>\n      <td>-0.020735</td>\n      <td>-0.112816</td>\n      <td>-0.048133</td>\n      <td>-0.023543</td>\n      <td>0.181349</td>\n      <td>0.036703</td>\n      <td>0.029212</td>\n    </tr>\n    <tr>\n      <th>887</th>\n      <td>0.262207</td>\n      <td>-0.051198</td>\n      <td>0.106294</td>\n      <td>0.046314</td>\n      <td>0.092977</td>\n      <td>-0.108752</td>\n      <td>-0.044362</td>\n      <td>-0.002959</td>\n      <td>0.155272</td>\n      <td>0.059984</td>\n      <td>...</td>\n      <td>0.073013</td>\n      <td>0.110786</td>\n      <td>0.063698</td>\n      <td>0.008327</td>\n      <td>-0.052947</td>\n      <td>-0.167440</td>\n      <td>-0.058933</td>\n      <td>0.194866</td>\n      <td>-0.023708</td>\n      <td>-0.078216</td>\n    </tr>\n    <tr>\n      <th>888</th>\n      <td>0.089673</td>\n      <td>-0.058084</td>\n      <td>0.181287</td>\n      <td>-0.083148</td>\n      <td>0.099980</td>\n      <td>-0.044123</td>\n      <td>-0.007683</td>\n      <td>0.061265</td>\n      <td>0.086505</td>\n      <td>0.138202</td>\n      <td>...</td>\n      <td>0.069641</td>\n      <td>0.003106</td>\n      <td>0.080451</td>\n      <td>-0.022685</td>\n      <td>-0.121224</td>\n      <td>-0.090532</td>\n      <td>-0.009335</td>\n      <td>0.123697</td>\n      <td>-0.010751</td>\n      <td>-0.057883</td>\n    </tr>\n    <tr>\n      <th>889</th>\n      <td>0.026983</td>\n      <td>-0.089072</td>\n      <td>0.177321</td>\n      <td>-0.059603</td>\n      <td>0.067363</td>\n      <td>-0.066032</td>\n      <td>-0.042090</td>\n      <td>0.006649</td>\n      <td>0.105672</td>\n      <td>0.130158</td>\n      <td>...</td>\n      <td>0.074123</td>\n      <td>-0.002361</td>\n      <td>0.087419</td>\n      <td>-0.017082</td>\n      <td>-0.152245</td>\n      <td>-0.093064</td>\n      <td>-0.028622</td>\n      <td>0.156932</td>\n      <td>0.014475</td>\n      <td>-0.028085</td>\n    </tr>\n  </tbody>\n</table>\n<p>890 rows × 300 columns</p>\n</div>",
      "text/plain": [
       "     token_avg0  token_avg1  token_avg2  token_avg3  token_avg4  token_avg5  \\\n",
       "0      0.154436    0.042025    0.106603   -0.018409    0.055064   -0.110473   \n",
       "1      0.146316   -0.049514    0.130602   -0.045639    0.176577   -0.024466   \n",
       "2      0.244598    0.058895    0.154219    0.048585    0.136651   -0.036705   \n",
       "3      0.188610   -0.048544    0.041357    0.004833    0.138942   -0.055199   \n",
       "4      0.156274   -0.046768    0.135716   -0.028251    0.143405   -0.055834   \n",
       "..          ...         ...         ...         ...         ...         ...   \n",
       "885    0.189939   -0.091825    0.069886   -0.041547    0.160320   -0.005779   \n",
       "886   -0.046209   -0.078544    0.176929   -0.086290    0.044155   -0.112407   \n",
       "887    0.262207   -0.051198    0.106294    0.046314    0.092977   -0.108752   \n",
       "888    0.089673   -0.058084    0.181287   -0.083148    0.099980   -0.044123   \n",
       "889    0.026983   -0.089072    0.177321   -0.059603    0.067363   -0.066032   \n",
       "\n",
       "     token_avg6  token_avg7  token_avg8  token_avg9  ...  token_avg290  \\\n",
       "0      0.039882    0.002460    0.129744    0.058930  ...      0.054728   \n",
       "1      0.055755    0.086503    0.113821    0.158433  ...      0.052342   \n",
       "2      0.037018    0.064442    0.118488    0.129758  ...      0.121074   \n",
       "3      0.036372    0.059581    0.126967    0.102629  ...      0.012302   \n",
       "4      0.020583    0.052493    0.124096    0.153234  ...      0.001162   \n",
       "..          ...         ...         ...         ...  ...           ...   \n",
       "885    0.064590    0.092563    0.062800    0.122943  ...      0.088572   \n",
       "886   -0.064324   -0.039424    0.138600    0.129153  ...      0.013887   \n",
       "887   -0.044362   -0.002959    0.155272    0.059984  ...      0.073013   \n",
       "888   -0.007683    0.061265    0.086505    0.138202  ...      0.069641   \n",
       "889   -0.042090    0.006649    0.105672    0.130158  ...      0.074123   \n",
       "\n",
       "     token_avg291  token_avg292  token_avg293  token_avg294  token_avg295  \\\n",
       "0        0.009576      0.050590     -0.022001     -0.033597     -0.091051   \n",
       "1        0.076558      0.056597     -0.020453     -0.081126     -0.084036   \n",
       "2        0.190364      0.090983     -0.070175     -0.104205     -0.147921   \n",
       "3        0.096600      0.010848     -0.010891     -0.035067     -0.102598   \n",
       "4        0.037805      0.047072     -0.028495     -0.064918     -0.107768   \n",
       "..            ...           ...           ...           ...           ...   \n",
       "885      0.095731      0.077700     -0.037176     -0.184658     -0.210214   \n",
       "886     -0.049760      0.096969     -0.020735     -0.112816     -0.048133   \n",
       "887      0.110786      0.063698      0.008327     -0.052947     -0.167440   \n",
       "888      0.003106      0.080451     -0.022685     -0.121224     -0.090532   \n",
       "889     -0.002361      0.087419     -0.017082     -0.152245     -0.093064   \n",
       "\n",
       "     token_avg296  token_avg297  token_avg298  token_avg299  \n",
       "0       -0.020110      0.163158     -0.010016     -0.018631  \n",
       "1       -0.028039      0.155239     -0.053626     -0.072127  \n",
       "2       -0.102687      0.291595     -0.087664     -0.141094  \n",
       "3       -0.024998      0.137281     -0.099979     -0.037174  \n",
       "4       -0.049309      0.141919     -0.051378     -0.076649  \n",
       "..            ...           ...           ...           ...  \n",
       "885     -0.025299      0.182387     -0.119329     -0.084234  \n",
       "886     -0.023543      0.181349      0.036703      0.029212  \n",
       "887     -0.058933      0.194866     -0.023708     -0.078216  \n",
       "888     -0.009335      0.123697     -0.010751     -0.057883  \n",
       "889     -0.028622      0.156932      0.014475     -0.028085  \n",
       "\n",
       "[890 rows x 300 columns]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(890, 333)\n"
     ]
    },
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>text</th>\n      <th>hasURL</th>\n      <th>urls</th>\n      <th>urls_expanded</th>\n      <th>hasUserURL</th>\n      <th>user_url</th>\n      <th>text_token</th>\n      <th>isNotOnlyText</th>\n      <th>Noun</th>\n      <th>Verb</th>\n      <th>...</th>\n      <th>token_avg290</th>\n      <th>token_avg291</th>\n      <th>token_avg292</th>\n      <th>token_avg293</th>\n      <th>token_avg294</th>\n      <th>token_avg295</th>\n      <th>token_avg296</th>\n      <th>token_avg297</th>\n      <th>token_avg298</th>\n      <th>token_avg299</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>EXTENDED: Dramatic video of gunfire inside hal...</td>\n      <td>1</td>\n      <td>[http://t.co/SbOu4rAp96]</td>\n      <td>[http://www.ctvnews.ca/video?clipId=472781&amp;pla...</td>\n      <td>1</td>\n      <td>http://t.co/1kTbzaumUY</td>\n      <td>[extended, dramatic, video, of, gunfire, insid...</td>\n      <td>1</td>\n      <td>7</td>\n      <td>1</td>\n      <td>...</td>\n      <td>0.054728</td>\n      <td>0.009576</td>\n      <td>0.050590</td>\n      <td>-0.022001</td>\n      <td>-0.033597</td>\n      <td>-0.091051</td>\n      <td>-0.020110</td>\n      <td>0.163158</td>\n      <td>-0.010016</td>\n      <td>-0.018631</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Police have clarified that there were two shoo...</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>1</td>\n      <td>http://t.co/sIkkpnZFhH</td>\n      <td>[police, have, clarified, that, there, were, t...</td>\n      <td>0</td>\n      <td>8</td>\n      <td>3</td>\n      <td>...</td>\n      <td>0.052342</td>\n      <td>0.076558</td>\n      <td>0.056597</td>\n      <td>-0.020453</td>\n      <td>-0.081126</td>\n      <td>-0.084036</td>\n      <td>-0.028039</td>\n      <td>0.155239</td>\n      <td>-0.053626</td>\n      <td>-0.072127</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Soldier killed in Ottawa identified as Cpl. Na...</td>\n      <td>1</td>\n      <td>[http://t.co/AOT1ZKyAei]</td>\n      <td>[http://ottawa.ctvnews.ca/video?clipId=473273&amp;...</td>\n      <td>1</td>\n      <td>http://ottawa.ctvnews.ca/</td>\n      <td>[soldier, killed, in, ottawa, identified, as, ...</td>\n      <td>1</td>\n      <td>3</td>\n      <td>2</td>\n      <td>...</td>\n      <td>0.121074</td>\n      <td>0.190364</td>\n      <td>0.090983</td>\n      <td>-0.070175</td>\n      <td>-0.104205</td>\n      <td>-0.147921</td>\n      <td>-0.102687</td>\n      <td>0.291595</td>\n      <td>-0.087664</td>\n      <td>-0.141094</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NORAD increases number of planes on higher ale...</td>\n      <td>1</td>\n      <td>[http://t.co/qsAnGNqBEw]</td>\n      <td>[http://cnn.it/1teSHUE]</td>\n      <td>1</td>\n      <td>http://t.co/kdkv08KSgi</td>\n      <td>[norad, increases, number, of, planes, on, hig...</td>\n      <td>1</td>\n      <td>7</td>\n      <td>3</td>\n      <td>...</td>\n      <td>0.012302</td>\n      <td>0.096600</td>\n      <td>0.010848</td>\n      <td>-0.010891</td>\n      <td>-0.035067</td>\n      <td>-0.102598</td>\n      <td>-0.024998</td>\n      <td>0.137281</td>\n      <td>-0.099979</td>\n      <td>-0.037174</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>All 3 patients injured in #OttawaShooting rele...</td>\n      <td>0</td>\n      <td>[]</td>\n      <td>[]</td>\n      <td>1</td>\n      <td>http://t.co/vAXH6cjeS8</td>\n      <td>[all, 3, patients, injured, in, ottawashooting...</td>\n      <td>0</td>\n      <td>4</td>\n      <td>3</td>\n      <td>...</td>\n      <td>0.001162</td>\n      <td>0.037805</td>\n      <td>0.047072</td>\n      <td>-0.028495</td>\n      <td>-0.064918</td>\n      <td>-0.107768</td>\n      <td>-0.049309</td>\n      <td>0.141919</td>\n      <td>-0.051378</td>\n      <td>-0.076649</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 333 columns</p>\n</div>",
      "text/plain": [
       "                                                text  hasURL  \\\n",
       "0  EXTENDED: Dramatic video of gunfire inside hal...       1   \n",
       "1  Police have clarified that there were two shoo...       0   \n",
       "2  Soldier killed in Ottawa identified as Cpl. Na...       1   \n",
       "3  NORAD increases number of planes on higher ale...       1   \n",
       "4  All 3 patients injured in #OttawaShooting rele...       0   \n",
       "\n",
       "                       urls  \\\n",
       "0  [http://t.co/SbOu4rAp96]   \n",
       "1                        []   \n",
       "2  [http://t.co/AOT1ZKyAei]   \n",
       "3  [http://t.co/qsAnGNqBEw]   \n",
       "4                        []   \n",
       "\n",
       "                                       urls_expanded  hasUserURL  \\\n",
       "0  [http://www.ctvnews.ca/video?clipId=472781&pla...           1   \n",
       "1                                                 []           1   \n",
       "2  [http://ottawa.ctvnews.ca/video?clipId=473273&...           1   \n",
       "3                            [http://cnn.it/1teSHUE]           1   \n",
       "4                                                 []           1   \n",
       "\n",
       "                    user_url  \\\n",
       "0     http://t.co/1kTbzaumUY   \n",
       "1     http://t.co/sIkkpnZFhH   \n",
       "2  http://ottawa.ctvnews.ca/   \n",
       "3     http://t.co/kdkv08KSgi   \n",
       "4     http://t.co/vAXH6cjeS8   \n",
       "\n",
       "                                          text_token  isNotOnlyText  Noun  \\\n",
       "0  [extended, dramatic, video, of, gunfire, insid...              1     7   \n",
       "1  [police, have, clarified, that, there, were, t...              0     8   \n",
       "2  [soldier, killed, in, ottawa, identified, as, ...              1     3   \n",
       "3  [norad, increases, number, of, planes, on, hig...              1     7   \n",
       "4  [all, 3, patients, injured, in, ottawashooting...              0     4   \n",
       "\n",
       "   Verb  ...  token_avg290  token_avg291  token_avg292  token_avg293  \\\n",
       "0     1  ...      0.054728      0.009576      0.050590     -0.022001   \n",
       "1     3  ...      0.052342      0.076558      0.056597     -0.020453   \n",
       "2     2  ...      0.121074      0.190364      0.090983     -0.070175   \n",
       "3     3  ...      0.012302      0.096600      0.010848     -0.010891   \n",
       "4     3  ...      0.001162      0.037805      0.047072     -0.028495   \n",
       "\n",
       "   token_avg294  token_avg295  token_avg296  token_avg297  token_avg298  \\\n",
       "0     -0.033597     -0.091051     -0.020110      0.163158     -0.010016   \n",
       "1     -0.081126     -0.084036     -0.028039      0.155239     -0.053626   \n",
       "2     -0.104205     -0.147921     -0.102687      0.291595     -0.087664   \n",
       "3     -0.035067     -0.102598     -0.024998      0.137281     -0.099979   \n",
       "4     -0.064918     -0.107768     -0.049309      0.141919     -0.051378   \n",
       "\n",
       "   token_avg299  \n",
       "0     -0.018631  \n",
       "1     -0.072127  \n",
       "2     -0.141094  \n",
       "3     -0.037174  \n",
       "4     -0.076649  \n",
       "\n",
       "[5 rows x 333 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df_test_avg.shape)\n",
    "df_test_avg.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Doc2Vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The Dimension of a vector is: 300\n"
     ]
    }
   ],
   "source": [
    "#Doc2vec 실행\n",
    "documents = [TaggedDocument(doc, [i]) for i, doc in enumerate(df_train['text_token'])]\n",
    "Doc2vec_model = Doc2Vec(vector_size=300, min_alpha=0.025, window=10, min_count=1, workers=4, epochs=120) #documents,\n",
    "Doc2vec_model.build_vocab(documents)\n",
    "\n",
    "Doc2vec_model.train(documents, epochs=Doc2vec_model.epochs, total_examples=Doc2vec_model.corpus_count)\n",
    "\n",
    "print(\"The Dimension of a vector is: {}\".format(len(Doc2vec_model.docvecs[0]))) # document vector의 정보"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "def most_similar_text(index):\n",
    "    similar = Doc2vec_model.docvecs.most_similar(index)\n",
    "\n",
    "    print(\"The quried text: \\n\\n{} \\n\\nis most similar to the text:\\n\\n{}\".format(df_train['text'][index],df_train['text'][similar[0][0]]))\n",
    "\n",
    "# most_similar_text(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "Doc2vec_model.save('./model/d2v_model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert to feature for df (Not Necessary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'Doc2vec_model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-29-94bc3e125de6>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mlist_doc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokens\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_train_X\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'text_token'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mlist_doc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mDoc2vec_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdocvecs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mdf_train_doc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist_doc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd_prefix\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'doc_vec'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'Doc2vec_model' is not defined"
     ]
    }
   ],
   "source": [
    "list_doc = []\n",
    "for index, tokens in enumerate(df_train['text_token']):\n",
    "    list_doc.append(Doc2vec_model.docvecs[index])\n",
    "    \n",
    "df_train_doc = pd.DataFrame(list_doc).add_prefix('doc_vec')\n",
    "df_train_doc = df_train.join(df_train_doc)\n",
    "df_train_doc.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Infer the document vectors from trained Doc2Vec Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "Doc2vec_model = Doc2Vec.load('./model/d2v_model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train['text_token_doc'] = copy.deepcopy(df_train_X['text_token'])\n",
    "\n",
    "for index, sentence in enumerate(df_train['text_token_doc']):\n",
    "    df_train['text_token_doc'][index] = Doc2vec_model.infer_vector(df_train['text_token_doc'][index],steps=50)\n",
    "\n",
    "# df_test_X[['text_token','text_token_doc']].head()\n",
    "df_train_doc = pd.DataFrame(df_train['text_token_doc'].values.tolist()).add_prefix('doc_vec')\n",
    "\n",
    "df_train_doc = df_train.join(df_train_doc).drop('text_token_doc',axis=1)\n",
    "# df_train_X_doc2.drop('text_token_doc', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test['text_token_doc'] = copy.deepcopy(df_test_X['text_token'])\n",
    "\n",
    "for index, sentence in enumerate(df_test['text_token_doc']):\n",
    "    df_test['text_token_doc'][index] = Doc2vec_model.infer_vector(df_test['text_token_doc'][index],steps=50)\n",
    "\n",
    "# df_test[['text_token','text_token_doc']].head()\n",
    "df_test_doc = pd.DataFrame(df_test['text_token_doc'].values.tolist()).add_prefix('doc_vec')\n",
    "\n",
    "df_test_doc = df_test.join(df_test_doc).drop('text_token_doc',axis=1)\n",
    "df_test.drop('text_token_doc', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Exporting data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train:  -> df_train_doc | df_train_avg -> 둘다 text/text_token을 드랍해야함\n",
    "# test: df_test -> df_test_doc | df_test_avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "# '''Target Variable set'''\n",
    "# df_train_y.to_csv('./data/train_y.csv', index = False)\n",
    "# df_test_y.to_csv('./data/test_y.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Word2Vec Averaged Vector Feature set'''\n",
    "df_train_avg.to_csv('./data/train_avg.csv', index = False)\n",
    "df_test_avg.to_csv('./data/test_avg.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Doc2vec Vector Feature set'''\n",
    "df_train_doc.to_csv('./data/train_doc.csv', index = False)\n",
    "df_test_doc.to_csv('./data/test_doc.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Export data without word vector'''\n",
    "data.to_csv('./data/X_basic.csv', index = False)\n",
    "data2.to_csv('./data/y_basic.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_avg = pd.concat([df_train_y,df_train_X_avg],axis=1)\n",
    "df_test_avg = pd.concat([df_test_y,df_test_X_avg],axis=1)\n",
    "df_train_doc = pd.concat([df_train_y,df_train_X_doc],axis=1)\n",
    "df_test_doc = pd.concat([df_test_y,df_test_X_doc],axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Data of Root tweets) X: 390 y: 390\n",
      "Number of total tokens appeared: 5637\n",
      "Number of unique tokens appeared: 1516\n",
      "\n",
      "Before fill: Does the dataset contain NaN value? True\n",
      "After fill: Does the dataset contain NaN value? False\n",
      "\n",
      "Dropping tweets with short length (<10)....\n",
      " (390, 34)\n",
      " -> (328, 34)\n"
     ]
    }
   ],
   "source": [
    "gurlitt_jsons = glob('../PHEME/all-rnr-annotated-threads/gurlitt-all-rnr-threads/**/source-tweets/*.json')\n",
    "ebolaessien_jsons = glob('../PHEME/all-rnr-annotated-threads/ebola-essien-all-rnr-threads/**/source-tweets/*.json')\n",
    "putinmissing_jsons = glob('../PHEME/all-rnr-annotated-threads/putinmissing-all-rnr-threads/**/source-tweets/*.json')\n",
    "\n",
    "added_files = [gurlitt_jsons, ebolaessien_jsons, putinmissing_jsons]\n",
    "\n",
    "valid_data = cross_val_jsons(added_files, False)\n",
    "data_lists, isRumorLists = extract_data(valid_data)\n",
    "X_valid = data_lists[0]\n",
    "y_valid = isRumorLists[0]\n",
    "print(\"(Data of Root tweets) X: {} y: {}\".format(len(X_valid),len(y_valid)))\n",
    "\n",
    "df_valid_X = pd.DataFrame(flatten_tweets(X_valid))\n",
    "df_valid = pd.concat([df_valid_X,y_valid],axis=1)\n",
    "df_valid[['has_question', 'has_exclaim', 'has_period','verified']] = df_valid[['has_question', 'has_exclaim', 'has_period','verified']].astype(int)\n",
    "\n",
    "for dataset in [df_valid]:\n",
    "    dataset['listed_count'].replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    # print(dataset['listed_count'].mean())\n",
    "    print(\"Before fill: Does the dataset contain NaN value? {}\".format(np.any(np.isnan(dataset['listed_count']))))\n",
    "    dataset['listed_count'].fillna(0,inplace=True)\n",
    "    print(\"After fill: Does the dataset contain NaN value? {}\".format(np.any(np.isnan(dataset['listed_count']))))\n",
    "print(\"\\nDropping tweets with short length (<10)....\\n\",df_valid.shape)\n",
    "df_valid.to_csv('./data/data_valid_notembeded.csv', index = False)\n",
    "\n",
    "df_valid.drop(df_valid[df_valid['word_count'] < 10].index, inplace=True)\n",
    "df_valid.reset_index(drop=True, inplace=True)\n",
    "print(\" ->\",df_valid.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec_ = w2v.Word2Vec.load('w2v_model')\n",
    "word_vectors = word2vec_.wv\n",
    "w2v_vectors = word_vectors.vectors # here you load vectors for each word in your model\n",
    "w2v_indices = {word: word_vectors.vocab[word].index for word in word_vectors.vocab} # here you load indices - with whom you can find an index of the particular word in your model \n",
    "\n",
    "df_valid['text_token_vec'] = copy.deepcopy(df_valid['text_token'])\n",
    "\n",
    "for index, sentence in enumerate(df_valid['text_token_vec']):\n",
    "    df_valid['text_token_vec'][index] = vectorize(sentence).mean(axis=0)\n",
    "\n",
    "avg = pd.DataFrame(df_valid['text_token_vec'].values.tolist()).add_prefix('token_avg')\n",
    "df_valid_avg = df_valid.join(avg).drop('text_token_vec',axis=1)\n",
    "\n",
    "df_valid.drop(['text_token_vec'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "Doc2vec_model = Doc2Vec.load('./model/d2v_model')\n",
    "df_valid['text_token_doc'] = copy.deepcopy(df_valid['text_token'])\n",
    "\n",
    "for index, sentence in enumerate(df_valid['text_token_doc']):\n",
    "    df_valid['text_token_doc'][index] = Doc2vec_model.infer_vector(df_valid['text_token_doc'][index],steps=50)\n",
    "\n",
    "doc = pd.DataFrame(df_valid['text_token_doc'].values.tolist()).add_prefix('doc_vec')\n",
    "df_valid_doc = df_valid.join(doc).drop('text_token_doc',axis=1)\n",
    "\n",
    "df_valid.drop(['text_token_doc'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid_avg.to_csv('./data/valid_avg.csv', index = False)\n",
    "df_valid_doc.to_csv('./data/valid_doc.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "x86VenvTest",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2
}