scattertext_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89d29fee",
   "metadata": {},
   "outputs": [],
   "source": [
    "Dataset URL: Corona Virus NLP Tweets data set\n",
    "https://www.kaggle.com/datasets/datatattle/covid-19-nlp-text-classification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "fa5e0348",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>UserName</th>\n",
       "      <th>ScreenName</th>\n",
       "      <th>Location</th>\n",
       "      <th>TweetAt</th>\n",
       "      <th>OriginalTweet</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3799</td>\n",
       "      <td>48751</td>\n",
       "      <td>London</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>@MeNyrbie @Phil_Gahan @Chrisitv https://t.co/i...</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3800</td>\n",
       "      <td>48752</td>\n",
       "      <td>UK</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>advice Talk to your neighbours family to excha...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3801</td>\n",
       "      <td>48753</td>\n",
       "      <td>Vagabonds</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>Coronavirus Australia: Woolworths to give elde...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3802</td>\n",
       "      <td>48754</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>My food stock is not the only one which is emp...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3803</td>\n",
       "      <td>48755</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16-03-2020</td>\n",
       "      <td>Me, ready to go at supermarket during the #COV...</td>\n",
       "      <td>Extremely Negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   UserName  ScreenName   Location     TweetAt  \\\n",
       "0      3799       48751     London  16-03-2020   \n",
       "1      3800       48752         UK  16-03-2020   \n",
       "2      3801       48753  Vagabonds  16-03-2020   \n",
       "3      3802       48754        NaN  16-03-2020   \n",
       "4      3803       48755        NaN  16-03-2020   \n",
       "\n",
       "                                       OriginalTweet           Sentiment  \n",
       "0  @MeNyrbie @Phil_Gahan @Chrisitv https://t.co/i...             Neutral  \n",
       "1  advice Talk to your neighbours family to excha...            Positive  \n",
       "2  Coronavirus Australia: Woolworths to give elde...            Positive  \n",
       "3  My food stock is not the only one which is emp...            Positive  \n",
       "4  Me, ready to go at supermarket during the #COV...  Extremely Negative  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "train_data = pd.read_csv('Corona_NLP_train.csv',  encoding='latin-1')\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "be278801",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OriginalTweet</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>@MeNyrbie @Phil_Gahan @Chrisitv https://t.co/i...</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>advice Talk to your neighbours family to excha...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Coronavirus Australia: Woolworths to give elde...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>My food stock is not the only one which is emp...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Me, ready to go at supermarket during the #COV...</td>\n",
       "      <td>Extremely Negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       OriginalTweet           Sentiment\n",
       "0  @MeNyrbie @Phil_Gahan @Chrisitv https://t.co/i...             Neutral\n",
       "1  advice Talk to your neighbours family to excha...            Positive\n",
       "2  Coronavirus Australia: Woolworths to give elde...            Positive\n",
       "3  My food stock is not the only one which is emp...            Positive\n",
       "4  Me, ready to go at supermarket during the #COV...  Extremely Negative"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = train_data.drop(['UserName', 'ScreenName', 'Location', 'TweetAt'], axis = 1)\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "134eaf13",
   "metadata": {},
   "outputs": [],
   "source": [
    "#train_data = train_data.reset_index(drop = True)\n",
    "#train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "bb7a97a7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        @menyrbie @phil_gahan @chrisitv https://t.co/i...\n",
       "1        advice talk neighbours family exchange phone n...\n",
       "2        coronavirus australia: woolworths give elderly...\n",
       "3        my food stock one empty... please, panic, ther...\n",
       "4        me, ready go supermarket #covid19 outbreak. no...\n",
       "                               ...                        \n",
       "41152    airline pilots offering stock supermarket shel...\n",
       "41153    response complaint provided citing covid-19 re...\n",
       "41154    you know itâs getting tough @kameronwilds rat...\n",
       "41155    is wrong smell hand sanitizer starting turn on...\n",
       "41156    @tartiicat well new/used rift s going $700.00 ...\n",
       "Name: OriginalTweet, Length: 41157, dtype: object"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Removing stop words, punctuation and converting to lowercase\n",
    "\n",
    "import nltk\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = stopwords.words('english')\n",
    "stopwordsandpunct = stop_words + list(string.punctuation)\n",
    "\n",
    "train_data['OriginalTweet'] = train_data['OriginalTweet'].apply(lambda w:' '.join(w for w in w.split() if w not in stopwordsandpunct))\n",
    "train_data['OriginalTweet'] = train_data['OriginalTweet'].str.lower()\n",
    "train_data['OriginalTweet']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f1393615",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                   @menyrbie @phil_gahan @chrisitv   \n",
       "1    advice talk neighbours family exchange phone n...\n",
       "2    coronavirus australia: woolworths give elderly...\n",
       "3    my food stock one empty... please, panic, ther...\n",
       "4    me, ready go supermarket #covid19 outbreak. no...\n",
       "Name: OriginalTweet, dtype: object"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Removing url\n",
    "import re\n",
    "def remove_url(tweet):\n",
    "    tweet = re.sub(r'\\w+:\\/{2}[\\d\\w-]+(\\.[\\d\\w-]+)*(?:(?:\\/[^\\s/]*))*', '', tweet)\n",
    "    return tweet\n",
    "\n",
    "train_data['OriginalTweet'] = train_data['OriginalTweet'].apply(remove_url)\n",
    "train_data['OriginalTweet'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1b52d81f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                                     \n",
       "1    advice talk neighbours family exchange phone n...\n",
       "2    coronavirus australia: woolworths give elderly...\n",
       "3    my food stock one empty... please, panic, ther...\n",
       "4    me, ready go supermarket  outbreak. not i'm pa...\n",
       "Name: OriginalTweet, dtype: object"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove mentions and hashtags\n",
    "def remove_mentions_hashs(tweet):\n",
    "    tweet = re.sub(\"@[A-Za-z0-9_]+\",\"\", tweet) #Remove mentions\n",
    "    tweet = re.sub(\"#[A-Za-z0-9_]+\",\"\", tweet) #Remove hashtags\n",
    "    return tweet\n",
    "\n",
    "train_data['OriginalTweet'] = train_data['OriginalTweet'].apply(remove_mentions_hashs)\n",
    "train_data['OriginalTweet'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "437f14f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import string\n",
    "def remove_empty_strings1(tweet):\n",
    "    tweet = re.sub(r\"^\\s+|\\s+$\", 'NaN', tweet)\n",
    "    return tweet\n",
    "train_data['OriginalTweet'] =train_data['OriginalTweet'].apply(remove_empty_strings1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "e76449f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OriginalTweet</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>advice talk neighbours family exchange phone n...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>coronavirus australia: woolworths give elderly...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>my food stock one empty... please, panic, ther...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>me, ready go supermarket  outbreak. not i'm pa...</td>\n",
       "      <td>Extremely Negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       OriginalTweet           Sentiment\n",
       "0                                                NaN             Neutral\n",
       "1  advice talk neighbours family exchange phone n...            Positive\n",
       "2  coronavirus australia: woolworths give elderly...            Positive\n",
       "3  my food stock one empty... please, panic, ther...            Positive\n",
       "4  me, ready go supermarket  outbreak. not i'm pa...  Extremely Negative"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import unicodedata\n",
    "def remove_nonascii(text):\n",
    "    text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')# apply compatibility decomposition\n",
    "    return text\n",
    "train_data['OriginalTweet'] = train_data['OriginalTweet'].apply(remove_nonascii)\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "243f1f8f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OriginalTweet</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Neutral</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>advice talk neighbours family exchange phone n...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>coronavirus australia: woolworths give elderly...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>my food stock one empty... please, panic, ther...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>me, ready go supermarket  outbreak. not i'm pa...</td>\n",
       "      <td>Extremely Negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       OriginalTweet           Sentiment\n",
       "0                                                NaN             Neutral\n",
       "1  advice talk neighbours family exchange phone n...            Positive\n",
       "2  coronavirus australia: woolworths give elderly...            Positive\n",
       "3  my food stock one empty... please, panic, ther...            Positive\n",
       "4  me, ready go supermarket  outbreak. not i'm pa...  Extremely Negative"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Removing emojis from tweets\n",
    "# https://stackoverflow.com/a/49146722/330558\n",
    "import re\n",
    "def remove_emojis(tweet):\n",
    "    pat =      re.compile(\"[\"\n",
    "                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                           u\"\\U00002702-\\U000027B0\"\n",
    "                           u\"\\U000024C2-\\U0001F251\"\n",
    "                           \"]+\", flags=re.UNICODE)\n",
    "    return pat.sub(r'', tweet)\n",
    "\n",
    "train_data['OriginalTweet'] =train_data['OriginalTweet'].apply(remove_emojis)\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "459f6b5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OriginalTweet</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>advice talk neighbours family exchange phone n...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>coronavirus australia: woolworths give elderly...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>my food stock one empty... please, panic, ther...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>me, ready go supermarket  outbreak. not i'm pa...</td>\n",
       "      <td>Extremely Negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>as news regionas first confirmed covid-19 case...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       OriginalTweet           Sentiment\n",
       "1  advice talk neighbours family exchange phone n...            Positive\n",
       "2  coronavirus australia: woolworths give elderly...            Positive\n",
       "3  my food stock one empty... please, panic, ther...            Positive\n",
       "4  me, ready go supermarket  outbreak. not i'm pa...  Extremely Negative\n",
       "5  as news regionas first confirmed covid-19 case...            Positive"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Removing all rows containing NaNs after removing hashtags, URLs\n",
    "train_data = train_data[train_data['OriginalTweet'] != 'NaN']\n",
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "b0523679",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OriginalTweet</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>advice talk neighbours family exchange phone n...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>coronavirus australia: woolworths give elderly...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>my food stock one empty... please, panic, ther...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>as news regionas first confirmed covid-19 case...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cashier grocery store sharing insights  to pro...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       OriginalTweet Sentiment\n",
       "1  advice talk neighbours family exchange phone n...  Positive\n",
       "2  coronavirus australia: woolworths give elderly...  Positive\n",
       "3  my food stock one empty... please, panic, ther...  Positive\n",
       "5  as news regionas first confirmed covid-19 case...  Positive\n",
       "6  cashier grocery store sharing insights  to pro...  Positive"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Extracting only the tweets that have sentiment labels - positive and negative \n",
    "pos_neg_tweets = train_data.loc[(train_data['Sentiment']=='Negative')|(train_data['Sentiment']=='Positive')]\n",
    "pos_neg_tweets.reset_index(drop = True)\n",
    "pos_neg_tweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "7fbdaa37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OriginalTweet</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[advice, talk, neighbours, family, exchange, p...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[coronavirus, australia:, woolworths, give, el...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[my, food, stock, one, empty..., please,, pani...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[as, news, regionas, first, confirmed, covid-1...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[cashier, grocery, store, sharing, insights, t...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       OriginalTweet Sentiment\n",
       "1  [advice, talk, neighbours, family, exchange, p...  Positive\n",
       "2  [coronavirus, australia:, woolworths, give, el...  Positive\n",
       "3  [my, food, stock, one, empty..., please,, pani...  Positive\n",
       "5  [as, news, regionas, first, confirmed, covid-1...  Positive\n",
       "6  [cashier, grocery, store, sharing, insights, t...  Positive"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Now we will perform tokenization\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import nltk\n",
    "from nltk import word_tokenize\n",
    "tokenizer = nltk.tokenize.WhitespaceTokenizer()\n",
    "def tokenize(text):\n",
    "    return tokenizer.tokenize(text)\n",
    "\n",
    "pos_neg_tweets['OriginalTweet'] = pos_neg_tweets['OriginalTweet'].apply(tokenize)\n",
    "pos_neg_tweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "91ae8ba6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OriginalTweet</th>\n",
       "      <th>Sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>advice, talk, neighbours, family, exchange, ph...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>coronavirus, australia:, woolworths, give, eld...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>my, food, stock, one, empty..., please,, panic...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>as, news, regionas, first, confirmed, covid-19...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cashier, grocery, store, sharing, insights, to...</td>\n",
       "      <td>Positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       OriginalTweet Sentiment\n",
       "1  advice, talk, neighbours, family, exchange, ph...  Positive\n",
       "2  coronavirus, australia:, woolworths, give, eld...  Positive\n",
       "3  my, food, stock, one, empty..., please,, panic...  Positive\n",
       "5  as, news, regionas, first, confirmed, covid-19...  Positive\n",
       "6  cashier, grocery, store, sharing, insights, to...  Positive"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "# converting token lists to strings using join() on each row or the OriginalTweet column\n",
    "pos_neg_tweets['OriginalTweet'] = pos_neg_tweets.loc[:, 'OriginalTweet'].apply(lambda x: ', '.join(x))\n",
    "pos_neg_tweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "8e715177",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21336, 2)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos_neg_tweets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d35becd7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OriginalTweet</th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>parsed</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>advice, talk, neighbours, family, exchange, ph...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>(advice, ,, talk, ,, neighbours, ,, family, ,,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>coronavirus, australia:, woolworths, give, eld...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>(coronavirus, ,, australia, :, ,, woolworths, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>my, food, stock, one, empty..., please,, panic...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>(my, ,, food, ,, stock, ,, one, ,, empty, ...,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>as, news, regionas, first, confirmed, covid-19...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>(as, ,, news, ,, regionas, ,, first, ,, confir...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>cashier, grocery, store, sharing, insights, to...</td>\n",
       "      <td>Positive</td>\n",
       "      <td>(cashier, ,, grocery, ,, store, ,, sharing, ,,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       OriginalTweet Sentiment  \\\n",
       "1  advice, talk, neighbours, family, exchange, ph...  Positive   \n",
       "2  coronavirus, australia:, woolworths, give, eld...  Positive   \n",
       "3  my, food, stock, one, empty..., please,, panic...  Positive   \n",
       "5  as, news, regionas, first, confirmed, covid-19...  Positive   \n",
       "6  cashier, grocery, store, sharing, insights, to...  Positive   \n",
       "\n",
       "                                              parsed  \n",
       "1  (advice, ,, talk, ,, neighbours, ,, family, ,,...  \n",
       "2  (coronavirus, ,, australia, :, ,, woolworths, ...  \n",
       "3  (my, ,, food, ,, stock, ,, one, ,, empty, ...,...  \n",
       "5  (as, ,, news, ,, regionas, ,, first, ,, confir...  \n",
       "6  (cashier, ,, grocery, ,, store, ,, sharing, ,,...  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Parsing OriginalTweet column with SpaCy's pipeline\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "def parsetweets(text):\n",
    "    text = text\n",
    "    parsedtext = nlp(text)\n",
    "    return parsedtext\n",
    "\n",
    "\n",
    "pos_neg_tweets['parsed'] = pos_neg_tweets.loc[:,'OriginalTweet'].apply(parsetweets)\n",
    "pos_neg_tweets.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "4e68f5ad",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <iframe\n",
       "            width=\"1000\"\n",
       "            height=\"800\"\n",
       "            src=\"scattertextdemo_plot.html\"\n",
       "            frameborder=\"0\"\n",
       "            allowfullscreen\n",
       "            \n",
       "        ></iframe>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.lib.display.IFrame at 0x2c18ab1cac0>"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import scattertext as st\n",
    "from IPython.display import IFrame, display, HTML\n",
    "from scattertext import CorpusFromPandas, produce_scattertext_explorer\n",
    "\n",
    "corpus = st.CorpusFromParsedDocuments(pos_neg_tweets.iloc[:1000,:], category_col ='Sentiment', parsed_col='parsed').build()\n",
    "html = st.produce_scattertext_explorer(corpus, category = 'Negative', category_name='Negative',\n",
    "                                      not_category_name='Positive', minimum_term_frequency=10,\n",
    "                                      width_in_pixels=1000, transform = st.Scalers.log_scale_standardize)\n",
    "file_name = 'scattertextdemo_plot.html'\n",
    "open(file_name, 'wb').write(html.encode('utf-8'))\n",
    "IFrame(src=file_name, width = 1000, height = 800)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc58a7e5",
   "metadata": {},
   "source": [
    "Above code produces a scatterplot which looks like this:\n",
    "\n",
    "<img src = 'scattertextplot1.png' width = 700 height = 500>\n",
    "\n",
    "We can see the words in two sentiment categories - positive and negative. Positive words are shown with red markers and negative with blue markers. Plot also has a list of top positive and top negative words on right side. With document count and word count information below the plot. We also get a search box, in which we can enter a word to look at information about the word. \n",
    "\n",
    "For example, we search for the word 'panic' and the result shows the frequency of this word per 25000 negative and positive terms and per 1000 documents, it also shows some mentions of the word in each tweet sentiment category. \n",
    "\n",
    "<img src = 'scatterplot_documentfrequenciespanic.png' width = 700 height = 500>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b855e91a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}