00_Initial_Data_Cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import json\n",
    "import time\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 19.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "data = \"dataset/Cell_Phones_and_Accessories_5.json.gz\"\n",
    "df_phones = pd.read_json(data, lines = True, compression = \"gzip\")\n",
    "df_phones.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load other datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 12.4 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "data = \"dataset/Video_Games_5.json.gz\"\n",
    "df_videogames = pd.read_json(data, lines = True, compression = \"gzip\")\n",
    "df_videogames.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 25.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "data = \"dataset/Automotive_5.json.gz\"\n",
    "df_automotive = pd.read_json(data, lines = True, compression = \"gzip\")\n",
    "df_automotive.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 19.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "data = \"dataset/Grocery_and_Gourmet_Food_5.json.gz\"\n",
    "df_grocery = pd.read_json(data, lines = True, compression = \"gzip\")\n",
    "df_grocery.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Combine Dataframes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(497577, 12)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_videogames.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1128437, 12)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_phones.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1711519, 12)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_automotive.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1143860, 12)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_grocery.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>asin</th>\n",
       "      <th>image</th>\n",
       "      <th>overall</th>\n",
       "      <th>reviewText</th>\n",
       "      <th>reviewTime</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>style</th>\n",
       "      <th>summary</th>\n",
       "      <th>unixReviewTime</th>\n",
       "      <th>verified</th>\n",
       "      <th>vote</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0700026657</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>This game is a bit hard to get the hang of, bu...</td>\n",
       "      <td>10 17, 2015</td>\n",
       "      <td>A1HP7NVNPFMA4N</td>\n",
       "      <td>Ambrosia075</td>\n",
       "      <td>NaN</td>\n",
       "      <td>but when you do it's great.</td>\n",
       "      <td>1445040000</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0700026657</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>I played it a while but it was alright. The st...</td>\n",
       "      <td>07 27, 2015</td>\n",
       "      <td>A1JGAP0185YJI6</td>\n",
       "      <td>travis</td>\n",
       "      <td>NaN</td>\n",
       "      <td>But in spite of that it was fun, I liked it</td>\n",
       "      <td>1437955200</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0700026657</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>ok game.</td>\n",
       "      <td>02 23, 2015</td>\n",
       "      <td>A1YJWEXHQBWK2B</td>\n",
       "      <td>Vincent G. Mezera</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Three Stars</td>\n",
       "      <td>1424649600</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0700026657</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>found the game a bit too complicated, not what...</td>\n",
       "      <td>02 20, 2015</td>\n",
       "      <td>A2204E1TH211HT</td>\n",
       "      <td>Grandma KR</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Two Stars</td>\n",
       "      <td>1424390400</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0700026657</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>great game, I love it and have played it since...</td>\n",
       "      <td>12 25, 2014</td>\n",
       "      <td>A2RF5B5H74JLPE</td>\n",
       "      <td>jon</td>\n",
       "      <td>NaN</td>\n",
       "      <td>love this game</td>\n",
       "      <td>1419465600</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         asin image  overall  \\\n",
       "0  0700026657   NaN        5   \n",
       "1  0700026657   NaN        4   \n",
       "2  0700026657   NaN        3   \n",
       "3  0700026657   NaN        2   \n",
       "4  0700026657   NaN        5   \n",
       "\n",
       "                                          reviewText   reviewTime  \\\n",
       "0  This game is a bit hard to get the hang of, bu...  10 17, 2015   \n",
       "1  I played it a while but it was alright. The st...  07 27, 2015   \n",
       "2                                           ok game.  02 23, 2015   \n",
       "3  found the game a bit too complicated, not what...  02 20, 2015   \n",
       "4  great game, I love it and have played it since...  12 25, 2014   \n",
       "\n",
       "       reviewerID       reviewerName style  \\\n",
       "0  A1HP7NVNPFMA4N        Ambrosia075   NaN   \n",
       "1  A1JGAP0185YJI6             travis   NaN   \n",
       "2  A1YJWEXHQBWK2B  Vincent G. Mezera   NaN   \n",
       "3  A2204E1TH211HT         Grandma KR   NaN   \n",
       "4  A2RF5B5H74JLPE                jon   NaN   \n",
       "\n",
       "                                       summary  unixReviewTime  verified vote  \n",
       "0                  but when you do it's great.      1445040000      True  NaN  \n",
       "1  But in spite of that it was fun, I liked it      1437955200     False  NaN  \n",
       "2                                  Three Stars      1424649600      True  NaN  \n",
       "3                                    Two Stars      1424390400      True  NaN  \n",
       "4                               love this game      1419465600      True  NaN  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_videogames.append(df_phones)\n",
    "df = df.append(df_automotive)\n",
    "df = df.append(df_grocery)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4481393, 12)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove rows with missing values from `reviewText`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.reviewText.notnull()].reindex()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Remove rows associated with non-verified reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.verified].reindex()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3942028, 12)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Definition of `X` and `y` (class to predict)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.reviewText.values\n",
    "y = df.overall.values > 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing/Normalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Emoticons Dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "emoticon_repl = {\n",
    "    # positive emoticons\n",
    "    r\":-?d+\": \" good \", # :D, :-D, :DD, ecc. in lower case\n",
    "    r\":[- ]?\\)+\": \" good \", # :-), :), :-)), :)), ecc\n",
    "    r\";-?\\)+\": \" good \", # ;). ;)). ;-), ecc.\n",
    "    r\"\\(+-?:\": \" good \", # (:, (-:, ecc.\n",
    "    r\"=\\)+\" : \" good \",\n",
    "    r\"\\b<3\\b\" : \" good \",    \n",
    "    # negative emoticons\n",
    "    r\"[\\s\\r\\t\\n]+:/+\": \" bad \", # :/\n",
    "    r\":\\\\+\": \" bad \", # :\\\n",
    "    r\"[\\s\\r\\t\\n]+\\)-?:\": \" bad \",  # ):, )):, )-:, ecc.\n",
    "    r\":-?\\(+\": \" bad \", # :(, :-(, :((, ecc.\n",
    "    r\"[\\s\\t\\r\\n]+d+-?:\": \" bad \"\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have used the following function to check if an emoticon is present in some review:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Check if re is present in some review\n",
    "def check_presence(regex, stop = 1, verbose = True):\n",
    "    j = 0\n",
    "    tv = False\n",
    "    for i in range(len(X)):\n",
    "        if re.search(regex, X[i].lower()):\n",
    "            tv = True\n",
    "            j += 1\n",
    "            sentiment = \"positive\" if y[i] else \"negative\"\n",
    "            if verbose:\n",
    "                print(\"Review #\",i)\n",
    "                print(\"Sentiment: {}\".format(sentiment))\n",
    "                print(\"Occurrences of the regex {} :\".format(regex), re.findall(regex, X[i].lower()))\n",
    "                print(\"Review text:\", X[i]+\"\\n\")\n",
    "            if j == stop:\n",
    "                break\n",
    "    return tv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Review # 161\n",
      "Sentiment: positive\n",
      "Occurrences of the regex :-?d+ : [':d']\n",
      "Review text: easy to put on and take off, good grip, cheap price, just don't rip it apart to see how strong it is and write a review about it and it's be fine :D\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_presence(r\":-?d+\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Review # 5841\n",
      "Sentiment: negative\n",
      "Occurrences of the regex [\\s\\r\\t\\n]+\\)-?: : [' ):']\n",
      "Review text: this product was not as described and  I returned it now they are giving me the run around on getting my money back. ):\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "check_presence(r\"[\\s\\r\\t\\n]+\\)-?:\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function for checking if an emoticon corresponds to the expected sentiment:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.stats import norm\n",
    "from scipy.stats import chi2_contingency"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_sentiment(regex):\n",
    "    pos = 0\n",
    "    neg = 0\n",
    "    lx = len(X)\n",
    "    \n",
    "    for i in range(lx):\n",
    "        if re.search(regex, X[i].lower()):\n",
    "            if y[i]:\n",
    "                pos += 1\n",
    "            else:\n",
    "                neg += 1\n",
    "                \n",
    "    tot_reviews = len(y)\n",
    "    tot_pos = np.sum(y)\n",
    "    tot_neg = tot_reviews - tot_pos\n",
    "                \n",
    "    n1 = pos + neg\n",
    "    n2 = tot_reviews - n1\n",
    "    \n",
    "    if emoticon_repl[regex] in [\" good \", \" great \"]:\n",
    "        test1 = pos\n",
    "        test2 = tot_pos - pos\n",
    "        sent = \"positive\"        \n",
    "    else:\n",
    "        test1 = neg\n",
    "        test2 = tot_neg - neg\n",
    "        sent = \"negative\"\n",
    "\n",
    "        \n",
    "    # One tailed proportion test\n",
    "    # https://online.stat.psu.edu/stat414/node/268/\n",
    "    p1 = test1/n1\n",
    "    p2 = test2/n2\n",
    "    p = (test1 + test2)/(n1+n2)\n",
    "    num = (p1-p2)\n",
    "    denom = np.sqrt(p*(1-p)*(1/n1 + 1/n2))\n",
    "    prop_zstat = num/denom\n",
    "    prop_pvalue = norm.sf(abs(prop_zstat))\n",
    "    \n",
    "    \n",
    "    # Chisq test\n",
    "    obs = np.array([[pos, neg],\n",
    "                    [(tot_pos-pos), (tot_neg-neg)]])\n",
    "    chi_zstat, chi_pvalue, _, _ = chi2_contingency(obs)\n",
    "    \n",
    "    cont_table = \"\\n{0:>16s} | No {0}\\nPos {1:12d} | {2:d}\\nNeg {3:12d} | {4:d}\\n\"\\\n",
    "    .format(regex, pos,tot_pos-pos, neg, tot_neg-neg)\n",
    "    \n",
    "    #\"Positive reviews containing regex {} : {}; p1 = {:.2f}\\n\".format(regex, pos, pos/n1) + \\\n",
    "    #\"Positive reviews not containing regex {} : {}; p2 = {:.2f}\\n\".format(regex, tot_pos-pos, (tot_pos-pos)/n2) + \\\n",
    "    print(\"Contingency table:\\n\" + \\\n",
    "          cont_table + \\\n",
    "          \"\\np1: #({} reviews) / #(reviews containing {}) = {:.2f}\\n\".format(sent, regex, test1/n1) + \\\n",
    "          \"p2: #({} reviews) / #(reviews not containing {}) = {:.2f}\\n\".format(sent, regex, test2/n2) + \\\n",
    "          \"H_0 proportion test: p1 - p2 <= 0\\n\" + \\\n",
    "          \"Proportion test p-value = {}\\n\".format(prop_pvalue) + \\\n",
    "          \n",
    "          \"Chisq test p-value = {}\".format(chi_pvalue))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For instance, we would expect that the proportion of positive reviews containing \":)\" and similar is larger than the proportion of positive reviews that do not contain \":)\".\n",
    "\n",
    "Similarly, we would expect that the proportion of negative reviews containing \":(\" and similar is larger than the proportion of negative reviews that do not contain \":(\".\n",
    "\n",
    "In order to check this, we test the significance of this difference for each emoticon in the dictionary:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Commented out for slowness and long output, interesting cases follow\n",
    "# for k in emoticon_repl:\n",
    "#     print(k)\n",
    "#     # check_sentiment_chisq(k)\n",
    "#     check_sentiment(k)\n",
    "#     print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Contingency table:\n",
      "\n",
      "       :[- ]?\\)+ | No :[- ]?\\)+\n",
      "Pos        26114 | 3290422\n",
      "Neg         1525 | 623967\n",
      "\n",
      "p1: #(positive reviews) / #(reviews containing :[- ]?\\)+) = 0.94\n",
      "p2: #(positive reviews) / #(reviews not containing :[- ]?\\)+) = 0.84\n",
      "H_0 proportion test: p1 - p2 <= 0\n",
      "Proportion test p-value = 0.0\n",
      "Chisq test p-value = 0.0\n"
     ]
    }
   ],
   "source": [
    "check_sentiment(r\":[- ]?\\)+\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are more negative reviews containing \"):\" and similar than positive ones, even though the difference is not that big:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Contingency table:\n",
      "\n",
      "[\\s\\r\\t\\n]+\\)-?: | No [\\s\\r\\t\\n]+\\)-?:\n",
      "Pos           18 | 3316518\n",
      "Neg           30 | 625462\n",
      "\n",
      "p1: #(negative reviews) / #(reviews containing [\\s\\r\\t\\n]+\\)-?:) = 0.62\n",
      "p2: #(negative reviews) / #(reviews not containing [\\s\\r\\t\\n]+\\)-?:) = 0.16\n",
      "H_0 proportion test: p1 - p2 <= 0\n",
      "Proportion test p-value = 4.675277707690704e-19\n",
      "Chisq test p-value = 5.376302524112476e-18\n"
     ]
    }
   ],
   "source": [
    "check_sentiment(r\"[\\s\\r\\t\\n]+\\)-?:\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The emoticon is very rare, so we delete it from the list:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "del emoticon_repl[r\"[\\s\\r\\t\\n]+\\)-?:\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The \"D:\" is very rare, and the difference in proportions is not significant."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Contingency table:\n",
      "\n",
      "[\\s\\t\\r\\n]+d+-?: | No [\\s\\t\\r\\n]+d+-?:\n",
      "Pos           28 | 3316508\n",
      "Neg           12 | 625480\n",
      "\n",
      "p1: #(negative reviews) / #(reviews containing [\\s\\t\\r\\n]+d+-?:) = 0.30\n",
      "p2: #(negative reviews) / #(reviews not containing [\\s\\t\\r\\n]+d+-?:) = 0.16\n",
      "H_0 proportion test: p1 - p2 <= 0\n",
      "Proportion test p-value = 0.007214829921059048\n",
      "Chisq test p-value = 0.02574679471755444\n"
     ]
    }
   ],
   "source": [
    "check_sentiment(r\"[\\s\\t\\r\\n]+d+-?:\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "del emoticon_repl[r\"[\\s\\t\\r\\n]+d+-?:\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Slang and abbreviations/acronyms Dictionary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Old approach: download list of internet acronyms and slang from http://www.netlingo.com/acronyms.php. Resulted in too many terms, with many corresponding to common words.\n",
    "\n",
    "Manual definition of a dictionary:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "slang_repl = {\n",
    "    \"tbh\" : \"to be honest\", \"afaik\" : \"as far as i know\",\n",
    "    \"ama\" : \"ask me anything\", \"b4\" : \"before\",\n",
    "    r\"baa+d\" : \" bad \", \"brb\" : \"be right back\",\n",
    "    \"btaim\" : \"be that as it may\", \"bts\" : \"behind the scenes\",\n",
    "    \"btw\" : \"by the way\", \"dyk\" : \"did you know\",\n",
    "    \"eli5\" : \"explain like i am five\", \"fomo\" : \"fear of missing out\",\n",
    "    \"ftfy\" : \"fixed that for you\", \"ftw\" : \"for the win\",\n",
    "    \"fyi\" : \"for your information\", \"g2g\" : \"got to go\",\n",
    "    r\"gooo+d\": \" good \", \"gtg\" : \"got to go\",\n",
    "    \"gg\" : \"good game\", r\"gr8t*\": \" great \",\n",
    "    \"gtr\" : \"got to run\", \"hmb\" : \"hit me back\",\n",
    "    \"hmu\" : \"hit me up\", \"hth\" : \"happy to help\",\n",
    "    \"icymi\" : \"in case you missed it\", \"idc\" : \"i do not care\",\n",
    "    \"idk\" : \"i do not know\", \"ikr\" : \"i know right\",\n",
    "    \"ily\" : \"i love you\", \"imho\" : \"in my humble opinion\",\n",
    "    \"imo\" : \"in my opinion\", \"irl\" : \"in real life\",\n",
    "    \"jk\" : \"just kidding\", \"lmao\" : \"laughing my ass off\",\n",
    "    \"lmk\" : \"let me know\", \"lol\" : \"laughing out loud\",\n",
    "    \"mfw\" : \"my face when\", \"nbd\" : \"no big deal\",\n",
    "    \"nm\" : \"not much\", \"nvm\" : \"never mind\",\n",
    "    \"omw\" : \"on my way\", \"op\" : \"original poster\",\n",
    "    \"ppl\" : \"people\", \"rofl\" : \"rolling on the floor laughing\",\n",
    "    \"roflmao\" : \"rolling on the floor laughing my ass off\", \"smh\" : \"shaking my head\",\n",
    "    \"tbbh\" : \"to be brutally honest\", \"tfw\" : \"that feeling when\",\n",
    "    \"til\" : \"today i learned\", \"tmi\" : \"too much information\",\n",
    "    \"wbu\" : \"what about you\", \"yolo\" : \"you only live once\"  \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Keep only slang that is present in the reviews\n",
    "# from tqdm.notebook import tqdm\n",
    "# \n",
    "# slang_keys = list(slang_repl.keys())\n",
    "# \n",
    "# present_slang = [k for k in tqdm(slang_keys) if check_presence(r\"\\b\"+k+r\"\\b\", verbose = False)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# slang_repl_subset = {k : slang_repl[k] for k in present_slang}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('slang_subset_manual.json', 'w') as fid:\n",
    "#     json.dump(slang_repl_subset, fid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('dataset/slang_subset_manual.json', 'r') as fid:\n",
    "    slang_repl_subset = json.load(fid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'btaim',\n",
       " 'dyk',\n",
       " 'eli5',\n",
       " 'ftfy',\n",
       " 'gtr',\n",
       " 'hmb',\n",
       " 'hmu',\n",
       " 'icymi',\n",
       " 'ikr',\n",
       " 'ily',\n",
       " 'mfw',\n",
       " 'omw',\n",
       " 'roflmao',\n",
       " 'tbbh',\n",
       " 'tfw',\n",
       " 'wbu'}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(slang_repl.keys())-set(slang_repl_subset.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "slang_repl = slang_repl_subset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Contracted forms"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will be interested in combinations of \"not\" + other terms and similar."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "contracted_repl = {\n",
    "    # casi particolari\n",
    "    r\"won\\'t\" : \"will not\", r\"won\\'\" : \"will not\",\n",
    "    r\"can\\'t\": \"can not\", r\"shan\\'t\": \"shall not\",\n",
    "    r\"shan\\'\": \"shall not\", r\"ain\\'t\": \"is not\",\n",
    "    r\"ain\\'\": \"is not\",\n",
    "    # casi generali\n",
    "    r\"n\\'t\": \" not\",\n",
    "    r\"n\\'\": \" not\",\n",
    "    r\"\\'s\": \" is\",\n",
    "    r\"\\'ve\": \" have\", \n",
    "    r\"\\'re\": \" are\", \n",
    "    r\"\\'ll\": \" will\", # Might also be \"shall\", in any case both will be considered stop words\n",
    "    r\"\\'d\": \" would\", # Might also be \"had\", in any case both will be considered stop words\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Complete preprocessing function"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Complete preprocessing function, including:\n",
    "\n",
    "- Lowering the text;\n",
    "- Removing video html part from reviews including a video;\n",
    "- Removing URLs;\n",
    "- Substituting emoticons with associated sentiment;\n",
    "- Translating slang and acronyms;\n",
    "- Expanding contracted forms;\n",
    "- Deleting non-alphanumeric characters (excluding the ones useful for tokenization);\n",
    "- Deleting terms consisting of digits only.\n",
    "\n",
    "Stop words removal and spelling correction are done after tokenization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(sent, translate_slang = True):\n",
    "    \n",
    "    sent = sent.lower()\n",
    "    sent = re.sub(r'^<div id=\"video.*>&nbsp;', '', sent) # Video-review part\n",
    "    sent = re.sub('https?://[A-Za-z0-9./]+', '', sent) # URLs\n",
    "    \n",
    "    for k in emoticon_repl:\n",
    "        sent = re.sub(k, emoticon_repl[k], sent)\n",
    "\n",
    "    if translate_slang:\n",
    "        for k in slang_repl:\n",
    "            sent = re.sub(r\"\\b\"+re.escape(k)+r\"\\b\", slang_repl[k], sent)\n",
    "            # sent = sent.replace(k, slang_repl[k])\n",
    "        \n",
    "    for k in contracted_repl:\n",
    "        sent = re.sub(k, contracted_repl[k], sent)\n",
    "    \n",
    "    sent = re.sub('[/]+', ' ', sent) # word1/word2 to word1 word2\n",
    "    # Remove non-alphanumeric characters (but not - and _, might be useful for tokenization)\n",
    "    sent = re.sub('[^A-Za-z0-9-_ ]+', '', sent)\n",
    "\n",
    "    # Remove words that are digits only\n",
    "    sent = re.sub('\\b\\d+\\b', '', sent)\n",
    "    \n",
    "    return sent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'A really neat and hard to find game that is different from the usual'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[1272]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a really neat and hard to find game that is different from the usual'"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess(X[1272])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Crashed in Vista.  Codemasters told me they don't support it in Windows 8.  Couldn't get it to work even after looking on the Internet.\""
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# X[1697]\n",
    "X[27]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'crashed in vista  codemasters told me they do not support it in windows 8  could not get it to work even after looking on the internet'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess(X[27])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 16min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df[\"reviewTextPreprocessed\"] = df[\"reviewText\"].apply(preprocess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1272                           Horrible...game was broken\n",
       "1272                                               Works!\n",
       "1272    This is our favorite flavor of all the differe...\n",
       "Name: reviewText, dtype: object"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"reviewText\"].loc[1272]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1272                              horriblegame was broken\n",
       "1272                                                works\n",
       "1272    this is our favorite flavor of all the differe...\n",
       "Name: reviewTextPreprocessed, dtype: object"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"reviewTextPreprocessed\"].loc[1272]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Delete rows where reviews are empty after preprocessing them:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~df.reviewTextPreprocessed.str.contains(r\"^\\s*$\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "98 reviews are empty after preprocessing:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3941930, 13)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "98"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(X)-len(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save cleaned dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[[\"reviewTextPreprocessed\", \"overall\", \"verified\"]].to_csv(\"dataset/preprocessed_dataset.csv\",\n",
    "                                                             index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}