Util_Parse.ipynb

{
 "metadata": {
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": 3
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2,
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coding: utf-8\n",
    "# Load dependencies for this Jupyter Notebook\n",
    "import os, json, errno\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sys import argv\n",
    "import string\n",
    "import time\n",
    "from util import to_unix_tmsp, parse_twitter_datetime\n",
    "from multiprocessing import Process\n",
    "\n",
    "\n",
    "#imports for text feature extraction:\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "import re\n",
    "from nltk.corpus import stopwords as stp\n",
    "from textblob import TextBlob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Tweets:\n",
    "    def __init__(self, event_name, output_dir=\"data/tweets\"):\n",
    "        self.event = event_name\n",
    "        self.data = {}\n",
    "        self.output_dir = output_dir\n",
    "        self.printable = set(string.printable)\n",
    "\n",
    "        utc_offset = {\n",
    "            \"germanwings-crash\": 1,\n",
    "            \"sydneysiege\": 11,\n",
    "            \"ottawashooting\": -4,\n",
    "            \"ferguson\":-5,\n",
    "            \"charliehebdo\":+1,\n",
    "        }\n",
    "        self.utc_offset = utc_offset[self.event]\n",
    "    \n",
    "    def append(self, twt, cat, thrd, is_src):\n",
    "        \"\"\" Convert tweet metadata into features.\n",
    "        Key to the `self.data` dictionary defined in this function define columns in\n",
    "        the CSV file produced by the `export` method.\n",
    "        Params:\n",
    "            - twt: The new tweet to add to the table\n",
    "            - cat: The category of the tweet, e.g. rumour\n",
    "            - thrd: The thread id of the tweet\n",
    "            - is_src : True if it's a source tweet and false if it is a reaction\n",
    "        \"\"\"\n",
    "        twt['category'] = cat\n",
    "        twt[\"thread\"] = thrd\n",
    "        twt[\"event\"] = self.event\n",
    "        twt[\"is_src\"] = is_src\n",
    "\n",
    "        twt_text=twt[\"text\"]\n",
    "        twt_text_filtered=str()\n",
    "        for c in twt_text:\n",
    "            if c in self.printable:\n",
    "                twt_text_filtered+=c\n",
    "\n",
    "        #print('twt text:',twt_text_filtered)\n",
    "        #print('type of twt_text', type(twt_text_filtered))\n",
    "        text_features=self.tweettext2features(twt_text_filtered)\n",
    "        \n",
    "        def get_utc_dist(obj):\n",
    "            offset = obj[\"user\"].get(\"utc_offset\")\n",
    "            conversion = 3600\n",
    "            return abs(self.utc_offset - offset / conversion) if offset else None\n",
    "\n",
    "        has_question = \"?\" in twt[\"text\"]\n",
    "        has_exclaim = \"!\" in twt[\"text\"]\n",
    "\n",
    "        features = {\n",
    "            # Thread metadata\n",
    "            \"is_rumor\": lambda obj : 1 if obj['category'] == \"rumours\" else 0,\n",
    "            \n",
    "            # Conservation metadata\n",
    "            \"thread\" : lambda obj : obj[\"thread\"],\n",
    "            \"in_reply_tweet\" : lambda obj : obj.get(\"in_reply_to_status_id\"),\n",
    "            \"event\" : lambda obj : obj.get(\"event\"),\n",
    "            \"tweet_id\" : lambda obj : obj.get(\"id\"),\n",
    "            \"is_source_tweet\" : lambda obj : 1 if twt[\"is_src\"] else 0,\n",
    "            \"in_reply_user\" : lambda obj : obj.get(\"in_reply_to_user_id\"),\n",
    "            \"user_id\" : lambda obj : obj[\"user\"].get(\"id\"),\n",
    "            \n",
    "            # Tweet metadata\n",
    "            \"tweet_length\": lambda obj : len(obj.get(\"text\",\"\")),\n",
    "            \"symbol_count\": lambda obj: len(obj[\"entities\"].get(\"symbols\", [])),\n",
    "            \"user_mentions\": lambda obj: len(obj[\"entities\"].get(\"user_mentions\", [])),\n",
    "            \"urls_count\": lambda obj : len(obj[\"entities\"].get(\"urls\", [])),\n",
    "            \"media_count\": lambda obj: len(obj[\"entities\"].get(\"media\", [])),\n",
    "            \"hashtags_count\": lambda obj : len(obj[\"entities\"].get(\"hashtags\", [])),\n",
    "            \"retweet_count\": lambda obj : obj.get(\"retweet_count\", 0),\n",
    "            \"favorite_count\": lambda obj : obj.get(\"favorite_count\"),\n",
    "            \"mentions_count\": lambda obj : len(obj[\"entities\"].get(\"user_mentions\", \"\")),\n",
    "            \"is_truncated\": lambda obj : 1 if obj.get(\"truncated\") else 0,\n",
    "            \"created\": lambda obj : self.datestr_to_tmsp(obj.get(\"created_at\")),\n",
    "            \"has_smile_emoji\": lambda obj: 1 if \"😊\" in obj[\"text\"] else 0,\n",
    "            \"sensitive\": lambda obj: 1 if obj.get(\"possibly_sensitive\") else 0,\n",
    "            \"has_place\": lambda obj: 1 if obj.get(\"place\") else 0,\n",
    "            \"has_coords\": lambda obj: 1 if obj.get(\"coordinates\") else 0,\n",
    "            \"has_quest\": lambda obj: 1 if has_question else 0,\n",
    "            \"has_exclaim\": lambda obj: 1 if has_exclaim else 0,\n",
    "            \"has_quest_or_exclaim\": lambda obj: 1 if (has_question or has_exclaim) else 0,\n",
    "\n",
    "            # User metadata\n",
    "            \"user.tweets_count\": lambda obj: obj[\"user\"].get(\"statuses_count\", 0),\n",
    "            \"user.verified\": lambda obj: 1 if obj[\"user\"].get(\"verified\") else 0,\n",
    "            \"user.followers_count\": lambda obj: obj[\"user\"].get(\"followers_count\"),\n",
    "            \"user.listed_count\": lambda obj: obj[\"user\"].get(\"listed_count\"),\n",
    "            \"user.desc_length\": lambda obj: len(obj[\"user\"].get(\"description\", \"\")),\n",
    "            \"user.handle_length\": lambda obj: len(obj[\"user\"].get(\"name\", \"\")),\n",
    "            \"user.name_length\": lambda obj: len(obj[\"user\"].get(\"screen_name\", \"\")),\n",
    "            \"user.notifications\": lambda obj: 1 if obj[\"user\"].get(\"notifications\") else 0,\n",
    "            \"user.friends_count\": lambda obj: obj[\"user\"].get(\"friends_count\"),\n",
    "            \"user.time_zone\": lambda obj: obj[\"user\"].get(\"time_zone\"),\n",
    "            \"user.desc_length\": lambda obj: len(obj[\"user\"][\"description\"]) if obj[\"user\"][\"description\"] else 0,\n",
    "            \"user.has_bg_img\": lambda obj: 1 if obj[\"user\"].get(\"profile_use_background_image\") else 0,\n",
    "            \"user.default_pic\": lambda obj: 1 if obj[\"user\"].get(\"default_profile\") else 0,\n",
    "            \"user.created_at\": lambda obj: self.datestr_to_tmsp(obj[\"user\"].get(\"created_at\")),\n",
    "            \"user.location\": lambda obj: 1 if obj[\"user\"].get(\"location\") else 0,\n",
    "            \"user.profile_sbcolor\": lambda obj: int(obj[\"user\"].get(\"profile_sidebar_border_color\"), 16),\n",
    "            \"user.profile_bgcolor\": lambda obj: int(obj[\"user\"].get(\"profile_background_color\"), 16),\n",
    "            \"user.utc_dist\": get_utc_dist,\n",
    "        }\n",
    "\n",
    "        for col in features:\n",
    "            self.data.setdefault(col, []).append(features[col](twt))\n",
    "\n",
    "        for col in text_features:\n",
    "            self.data.setdefault(col, []).append(text_features[col])\n",
    "\n",
    "    def tweettext2features(self, tweet_text):   \n",
    "        \"\"\" Extracts some text features from the text of each tweet. The extracted features are as follows:\n",
    "        hasperiod: has period\n",
    "        number_punct: number of punctuation marks\n",
    "        negativewordcount: the count of the defined negative word counts\n",
    "        positivewordcount :the count of the defined positive word counts\n",
    "        capitalratio: ratio of capital letters to all the letters\n",
    "        contentlength: length of text\n",
    "        sentimentscore: sentiment score by textBlob\n",
    "        Noun: number of nouns\n",
    "        Verb: number of verbs\n",
    "        Adjective: number of adjectives\n",
    "        Pronoun: number of pronouns\n",
    "        Adverb: number of adverbs\n",
    "        Param:\n",
    "            - tweet_text: text of tweet\n",
    "        Return: a dict containing the mentioned text features\n",
    "        \"\"\"\n",
    "        #punctuations\n",
    "        def punctuationanalysis(tweet_text):\n",
    "            punctuations= [\"\\\"\",\"(\",\")\",\"*\",\",\",\"-\",\"_\",\".\",\"~\",\"%\",\"^\",\"&\",\"!\",\"#\",'@'\n",
    "               \"=\",\"\\'\",\"\\\\\",\"+\",\"/\",\":\",\"[\",\"]\",\"«\",\"»\",\"،\",\"؛\",\"?\",\".\",\"…\",\"$\",\n",
    "               \"|\",\"{\",\"}\",\"٫\",\";\",\">\",\"<\",\"1\",\"2\",\"3\",\"4\",\"5\",\"6\",\"7\",\"8\",\"9\",\"0\"]\n",
    "            hasperiod=sum(c =='.' for c in tweet_text)\n",
    "            number_punct=sum(c in punctuations for c in tweet_text)\n",
    "            return {'hasperiod':hasperiod,'number_punct':number_punct}\n",
    "\n",
    "        def negativewordcount(tokens):\n",
    "            count = 0\n",
    "            negativeFeel = ['tired', 'sick', 'bord', 'uninterested', 'nervous', 'stressed',\n",
    "                            'afraid', 'scared', 'frightened', 'boring','bad',\n",
    "                            'distress', 'uneasy', 'angry', 'annoyed', 'pissed',\"hate\",\n",
    "                            'sad', 'bitter', 'down', 'depressed', 'unhappy','heartbroken','jealous', 'fake', 'stupid', 'strange','absurd', 'crazy']\n",
    "            for negative in negativeFeel:\n",
    "                if negative in tokens:\n",
    "                    count += 1\n",
    "            return count\n",
    "\n",
    "        def positivewordcount(tokens):\n",
    "            count = 0\n",
    "            positivewords = ['joy', ' happy', 'hope', 'kind', 'surprise'\n",
    "                            , 'excite', ' interest', 'admire',\"delight\",\"yummy\",\n",
    "                            'confidenc', 'good', 'satisf', 'pleasant',\n",
    "                            'proud', 'amus', 'amazing', 'awesome',\"love\",\"passion\",\"great\",\"like\",\"wow\",\"delicious\", \"true\", \"correct\", \"crazy\"]\n",
    "            for pos in positivewords:\n",
    "                if pos in tokens:\n",
    "                    count += 1\n",
    "            return count\n",
    "\n",
    "        def capitalratio(tweet_text):\n",
    "            uppers = [l for l in tweet_text if l.isupper()]\n",
    "            capitalratio = len(uppers) / len(tweet_text)\n",
    "            return capitalratio\n",
    "\n",
    "        def contentlength(words):\n",
    "            wordcount = len(words)\n",
    "            return wordcount\n",
    "\n",
    "        def sentimentscore(tweet_text):\n",
    "            analysis = TextBlob(tweet_text)\n",
    "            return analysis.sentiment.polarity\n",
    "\n",
    "        def getposcount(tweet_text):\n",
    "            postag = []\n",
    "            poscount = {}\n",
    "            poscount['Noun']=0\n",
    "            poscount['Verb']=0\n",
    "            poscount['Adjective'] = 0\n",
    "            poscount['Pronoun']=0\n",
    "            poscount['FirstPersonPronoun']=0\n",
    "            poscount['SecondPersonPronoun']=0\n",
    "            poscount['ThirdPersonPronoun']=0\n",
    "            poscount['Adverb']=0\n",
    "            Nouns = {'NN','NNS','NNP','NNPS'}\n",
    "            Verbs={'VB','VBP','VBZ','VBN','VBG','VBD','To'}\n",
    "            first_person_pronouns=['I','me','my','mine','we','us','our','ours']\n",
    "            second_person_pronouns=['you','your','yours']\n",
    "            third_person_pronouns=['he','she','it','him','her','it','his','hers','its','they','them','their','theirs']\n",
    "\n",
    "            word_tokens = nltk.word_tokenize(re.sub(r'([^\\s\\w]|_)+', '', tweet_text))\n",
    "            for word in word_tokens:\n",
    "                w_lower=word.lower()\n",
    "                if w_lower in first_person_pronouns:\n",
    "                    poscount['FirstPersonPronoun']+=1\n",
    "                elif w_lower in second_person_pronouns:\n",
    "                    poscount['SecondPersonPronoun']+=1\n",
    "                elif w_lower in third_person_pronouns:\n",
    "                    poscount['ThirdPersonPronoun']+=1\n",
    "\n",
    "            postag = nltk.pos_tag(word_tokens)\n",
    "            for g1 in postag:\n",
    "                if g1[1] in Nouns:\n",
    "                    poscount['Noun'] += 1\n",
    "                elif g1[1] in Verbs:\n",
    "                    poscount['Verb']+= 1\n",
    "                elif g1[1]=='ADJ'or g1[1]=='JJ':\n",
    "                    poscount['Adjective']+=1\n",
    "                elif g1[1]=='PRP' or g1[1]=='PRON':\n",
    "                    poscount['Pronoun']+=1\n",
    "                elif g1[1]=='ADV':\n",
    "                    poscount['Adverb']+=1\n",
    "            return poscount\n",
    "        def tweets2tokens(tweet_text):\n",
    "            tokens = nltk.word_tokenize(re.sub(r'([^\\s\\w]|_)+','', tweet_text.lower()))\n",
    "            url=0\n",
    "            for token in tokens:\n",
    "                if token.startswith( 'http' ):\n",
    "                    url=1\n",
    "\n",
    "            return tokens,url\n",
    "\n",
    "\n",
    "        # the code for def tweettext2features(tweet_text):\n",
    "        features=dict()\n",
    "\n",
    "        tokens,url=tweets2tokens(tweet_text)\n",
    "\n",
    "        punc_dict=punctuationanalysis(tweet_text)\n",
    "        features.update(punc_dict)\n",
    "        features['negativewordcount']=(negativewordcount(tokens))\n",
    "        features['positivewordcount']=(positivewordcount(tokens))\n",
    "        features['capitalratio']=(capitalratio(tweet_text))\n",
    "        features['contentlength']=(contentlength(tokens))\n",
    "        features['sentimentscore']=(sentimentscore(tweet_text))\n",
    "        pos_dict=getposcount(tweet_text)\n",
    "        features.update(pos_dict)\n",
    "        features['has_url_in_text']=(url)\n",
    "        # print(\"features\",features)\n",
    "        return features\n",
    "\n",
    "    def export(self):\n",
    "        fn = \"%s/%s.csv\" % (self.output_dir, self.event)\n",
    "        df = pd.DataFrame(data=self.data)\n",
    "        df.to_csv(fn, index=False)\n",
    "        return fn\n",
    "    \n",
    "    def datestr_to_tmsp(self, datestr):\n",
    "        \"\"\" Converts Twitter's datetime format to Unix timestamp \n",
    "        Param:\n",
    "            - datestr: datetime string, e.g. Mon Dec 10 4:12:32.33 +7000 2018\n",
    "        Return: Unix timestamp\n",
    "        \"\"\"\n",
    "        return to_unix_tmsp([parse_twitter_datetime(datestr)])[0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pheme_to_csv(event, Parser=Tweets, output=\"data/tweets\"):\n",
    "    \"\"\" Parses json data stored in directories of the PHEME dataset into a CSV file.\n",
    "    \n",
    "    Params:\n",
    "        - event: Name fake news event and directory name in PHEME dataset\n",
    "    \n",
    "    Return: None\n",
    "    \"\"\"\n",
    "    start = time.time()\n",
    "    data = Parser(event, output_dir=output)\n",
    "    dataset = \"raw/pheme-rnr-dataset\"\n",
    "    thread_number = 0         \n",
    "    for category in os.listdir(\"%s/%s\" % (dataset, event)):\n",
    "        print('event:',event,'category:',category,category=='rumours')\n",
    "        for thread in os.listdir(\"%s/%s/%s\" % (dataset, event, category)):\n",
    "            with open(\"%s/%s/%s/%s/source-tweet/%s.json\" % (dataset, event, category, thread, thread)) as f:\n",
    "                tweet = json.load(f)\n",
    "            data.append(tweet, category, thread, True)\n",
    "            thread_number += 1\n",
    "            for reaction in os.listdir(\"%s/%s/%s/%s/reactions\" % (dataset, event, category, thread)):\n",
    "                with open(\"%s/%s/%s/%s/reactions/%s\" % (dataset, event, category, thread, reaction)) as f:\n",
    "                    tweet = json.load(f)\n",
    "                data.append(tweet, category, thread, False)\n",
    "    fn = data.export()\n",
    "    print(\"%s was generated in %s minutes\" % (fn, (time.time() - start) / 60))\n",
    "    return None\n",
    "\n",
    "def agg_event_data(df, limit=0):\n",
    "    \"\"\" Aggregate tabular tweet data from a PHEME event into aggregated thread-level data\n",
    "    \n",
    "    Params:\n",
    "        - df: the DataFrame with tabular tweet data\n",
    "       \n",
    "    Return: A DataFrame with thread-level data for this event\n",
    "    \"\"\"\n",
    "    data = df.head(limit) if limit > 0 else df\n",
    "    data = data.replace({\"has_url\": {\"True\": True, \"False\": False}})\n",
    "    agg = data.groupby(\"thread\") \\\n",
    "        .agg({\"favorite_count\": sum,\n",
    "              \"retweet_count\": sum,\n",
    "              \"is_rumor\": max,\n",
    "              \"has_url\": lambda col: np.count_nonzero(col) / len(col),\n",
    "              \"id\": len,\n",
    "              \"hashtags_count\": lambda col: len([True for total in col if total > 0]) / len(col),\n",
    "              \"text\": lambda col: len([True for txt in col if \"😊\" in txt]) / len(col)}) \\\n",
    "        .rename(columns={\"favorite_count\": \"favorite_total\",\n",
    "                         \"retweet_count\": \"retweet_total\",\n",
    "                         \"user.friends_count\": \"friends_total\",\n",
    "                         \"id\": \"thread_length\",\n",
    "                         \"has_url\":\"url_proportion\",\n",
    "                         \"hashtags_count\": \"hashtag_proportion\",\n",
    "                         \"text\": \"smile_emoji_proportion\"})\n",
    "    src = data[data[\"thread\"] == data[\"id\"]][[\"thread\", \"user.followers_count\"]]  # source tweets will have equal thread id and tweet id\n",
    "    src = src.rename(columns={\"user.followers_count\": \"src_followers_count\"})\n",
    "    thrd_data = pd.merge(agg, src, on=\"thread\")\n",
    "    return thrd_data\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    print(\"Running %s to parse %s\" % (argv[0], argv[1]))\n",
    "    if(argv[1]==\"all\"):\n",
    "        events=[\n",
    "            \"germanwings-crash\",\n",
    "            \"sydneysiege\",\n",
    "            \"ottawashooting\",\n",
    "            \"ferguson\",\n",
    "            #\"charliehebdo\",\n",
    "        ]\n",
    "        dataset = \"../raw/pheme-rnr-dataset\"\n",
    "        processes=[]\n",
    "        for event in events:\n",
    "            p=Process(target=pheme_to_csv,args=(event,))\n",
    "            p.start()\n",
    "            processes.append(p)\n",
    "            #pheme_to_csv(event)\n",
    "        for p in processes:\n",
    "            p.join()\n",
    "            \n",
    "    else:\n",
    "        pheme_to_csv(argv[1])"
   ]
  }
 ]
}