6_final_merge.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\aniqa\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\aniqa\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package vader_lexicon to\n",
      "[nltk_data]     C:\\Users\\aniqa\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package vader_lexicon is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>cleaned_review</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AGTJAM6NFSYTR5KB6DQNHZ3ZIFFQ</td>\n",
       "      <td>positive</td>\n",
       "      <td>awesome</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AE7BV6IMNPZ3F266H7PXMH3BZQNQ</td>\n",
       "      <td>positive</td>\n",
       "      <td>best elvis golden records series</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AE7BV6IMNPZ3F266H7PXMH3BZQNQ</td>\n",
       "      <td>neutral</td>\n",
       "      <td>exgi elvis tearing studio b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AE7BV6IMNPZ3F266H7PXMH3BZQNQ</td>\n",
       "      <td>neutral</td>\n",
       "      <td>elvis rhythm country essential absolutely</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AE7BV6IMNPZ3F266H7PXMH3BZQNQ</td>\n",
       "      <td>positive</td>\n",
       "      <td>late great christine mcvie remembered celebrated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        user_id sentiment  \\\n",
       "0  AGTJAM6NFSYTR5KB6DQNHZ3ZIFFQ  positive   \n",
       "1  AE7BV6IMNPZ3F266H7PXMH3BZQNQ  positive   \n",
       "2  AE7BV6IMNPZ3F266H7PXMH3BZQNQ   neutral   \n",
       "3  AE7BV6IMNPZ3F266H7PXMH3BZQNQ   neutral   \n",
       "4  AE7BV6IMNPZ3F266H7PXMH3BZQNQ  positive   \n",
       "\n",
       "                                     cleaned_review  \n",
       "0                                           awesome  \n",
       "1                  best elvis golden records series  \n",
       "2                       exgi elvis tearing studio b  \n",
       "3         elvis rhythm country essential absolutely  \n",
       "4  late great christine mcvie remembered celebrated  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "from sklearn.calibration import LabelEncoder\n",
    "\n",
    "# Download necessary NLTK data\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')\n",
    "nltk.download('vader_lexicon')\n",
    "\n",
    "# Load the data\n",
    "df = pd.read_csv('merged_data.csv')\n",
    "\n",
    "# Data cleaning function\n",
    "def clean_text(text):\n",
    "    text = re.sub(r'[^\\w\\s]', '', text)\n",
    "    text = text.lower()\n",
    "    tokens = word_tokenize(text)\n",
    "    tokens = [word for word in tokens if word not in stopwords.words('english')]\n",
    "    return ' '.join(tokens)\n",
    "\n",
    "# Clean review text\n",
    "df['cleaned_review'] = df['review_title'].apply(clean_text)\n",
    "\n",
    "# Sentiment analysis function using VADER\n",
    "sid = SentimentIntensityAnalyzer()\n",
    "\n",
    "def get_sentiment(text):\n",
    "    scores = sid.polarity_scores(text)\n",
    "    return 'positive' if scores['compound'] >= 0.05 else ('negative' if scores['compound'] <= -0.05 else 'neutral')\n",
    "\n",
    "# Apply sentiment analysis to get sentiment labels\n",
    "df['sentiment'] = df['cleaned_review'].apply(get_sentiment)\n",
    "\n",
    "# Encode sentiment labels\n",
    "le = LabelEncoder()\n",
    "df['sentiment_encoded'] = le.fit_transform(df['sentiment'])\n",
    "\n",
    "# Display the first few rows to verify the results\n",
    "df[['user_id', 'sentiment', 'cleaned_review']].head()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Positive Reviews Summary:\n",
      "              rating   helpful_vote  verified_purchase          price  \\\n",
      "count  413701.000000  413701.000000      413701.000000  413701.000000   \n",
      "mean        4.613922       0.298704           0.850252      44.060923   \n",
      "std         0.900461       1.440988           0.356825      92.121921   \n",
      "min         1.000000       0.000000           0.000000       0.010000   \n",
      "25%         5.000000       0.000000           1.000000      13.990000   \n",
      "50%         5.000000       0.000000           1.000000      22.000000   \n",
      "75%         5.000000       0.000000           1.000000      39.990000   \n",
      "max         5.000000     274.000000           1.000000   15499.950000   \n",
      "\n",
      "       sentiment_encoded  review_length     word_count  \n",
      "count           413701.0  413701.000000  413701.000000  \n",
      "mean                 2.0      17.015685       2.795599  \n",
      "std                  0.0      11.372258       1.674108  \n",
      "min                  2.0       2.000000       1.000000  \n",
      "25%                  2.0       9.000000       2.000000  \n",
      "50%                  2.0      14.000000       2.000000  \n",
      "75%                  2.0      22.000000       4.000000  \n",
      "max                  2.0      95.000000      18.000000  \n",
      "\n",
      "Neutral Reviews Summary:\n",
      "              rating   helpful_vote  verified_purchase          price  \\\n",
      "count  210278.000000  210278.000000      210278.000000  210278.000000   \n",
      "mean        3.690914       0.309633           0.891748      44.617402   \n",
      "std         1.616409       2.751996           0.310699      89.943441   \n",
      "min         1.000000       0.000000           0.000000       0.010000   \n",
      "25%         2.000000       0.000000           1.000000      13.990000   \n",
      "50%         5.000000       0.000000           1.000000      21.990000   \n",
      "75%         5.000000       0.000000           1.000000      39.990000   \n",
      "max         5.000000    1030.000000           1.000000    3899.000000   \n",
      "\n",
      "       sentiment_encoded  review_length     word_count  \n",
      "count           210278.0  210278.000000  210278.000000  \n",
      "mean                 1.0      15.135991       2.440997  \n",
      "std                  0.0      10.445918       1.612164  \n",
      "min                  1.0       0.000000       0.000000  \n",
      "25%                  1.0       8.000000       1.000000  \n",
      "50%                  1.0      13.000000       2.000000  \n",
      "75%                  1.0      19.000000       3.000000  \n",
      "max                  1.0      96.000000      20.000000  \n",
      "\n",
      "Negative Reviews Summary:\n",
      "             rating  helpful_vote  verified_purchase         price  \\\n",
      "count  63208.000000  63208.000000       63208.000000  63208.000000   \n",
      "mean       2.184201      0.340811           0.892403     49.623973   \n",
      "std        1.440474      1.472604           0.309873    100.817412   \n",
      "min        1.000000      0.000000           0.000000      0.010000   \n",
      "25%        1.000000      0.000000           1.000000     14.980000   \n",
      "50%        2.000000      0.000000           1.000000     23.990000   \n",
      "75%        3.000000      0.000000           1.000000     44.950000   \n",
      "max        5.000000     83.000000           1.000000   3849.000000   \n",
      "\n",
      "       sentiment_encoded  review_length    word_count  \n",
      "count            63208.0   63208.000000  63208.000000  \n",
      "mean                 0.0      20.123086      3.164330  \n",
      "std                  0.0      12.595304      1.924197  \n",
      "min                  0.0       2.000000      1.000000  \n",
      "25%                  0.0      12.000000      2.000000  \n",
      "50%                  0.0      17.000000      3.000000  \n",
      "75%                  0.0      26.000000      4.000000  \n",
      "max                  0.0      93.000000     17.000000  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>sentiment</th>\n",
       "      <th>review_length</th>\n",
       "      <th>word_count</th>\n",
       "      <th>helpful_vote</th>\n",
       "      <th>verified_purchase</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AGTJAM6NFSYTR5KB6DQNHZ3ZIFFQ</td>\n",
       "      <td>positive</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AE7BV6IMNPZ3F266H7PXMH3BZQNQ</td>\n",
       "      <td>positive</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AE7BV6IMNPZ3F266H7PXMH3BZQNQ</td>\n",
       "      <td>neutral</td>\n",
       "      <td>27</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AE7BV6IMNPZ3F266H7PXMH3BZQNQ</td>\n",
       "      <td>neutral</td>\n",
       "      <td>41</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AE7BV6IMNPZ3F266H7PXMH3BZQNQ</td>\n",
       "      <td>positive</td>\n",
       "      <td>48</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        user_id sentiment  review_length  word_count  \\\n",
       "0  AGTJAM6NFSYTR5KB6DQNHZ3ZIFFQ  positive              7           1   \n",
       "1  AE7BV6IMNPZ3F266H7PXMH3BZQNQ  positive             32           5   \n",
       "2  AE7BV6IMNPZ3F266H7PXMH3BZQNQ   neutral             27           5   \n",
       "3  AE7BV6IMNPZ3F266H7PXMH3BZQNQ   neutral             41           5   \n",
       "4  AE7BV6IMNPZ3F266H7PXMH3BZQNQ  positive             48           6   \n",
       "\n",
       "   helpful_vote  verified_purchase  \n",
       "0             1                  1  \n",
       "1             0                  0  \n",
       "2             0                  0  \n",
       "3             0                  0  \n",
       "4             0                  0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Feature Engineering: Extract relevant features\n",
    "df['review_length'] = df['cleaned_review'].apply(len)\n",
    "df['word_count'] = df['cleaned_review'].apply(lambda x: len(x.split()))\n",
    "df['helpful_vote'] = df['helpful_vote'].fillna(0)\n",
    "df['verified_purchase'] = df['verified_purchase'].astype(int)\n",
    "\n",
    "# Segment customers based on sentiment\n",
    "positive_reviews = df[df['sentiment'] == 'positive']\n",
    "neutral_reviews = df[df['sentiment'] == 'neutral']\n",
    "negative_reviews = df[df['sentiment'] == 'negative']\n",
    "\n",
    "# Analyze characteristics of each segment\n",
    "positive_summary = positive_reviews.describe()\n",
    "neutral_summary = neutral_reviews.describe()\n",
    "negative_summary = negative_reviews.describe()\n",
    "\n",
    "# Display summaries\n",
    "print(\"Positive Reviews Summary:\")\n",
    "print(positive_summary)\n",
    "print(\"\\nNeutral Reviews Summary:\")\n",
    "print(neutral_summary)\n",
    "print(\"\\nNegative Reviews Summary:\")\n",
    "print(negative_summary)\n",
    "\n",
    "display(df[['user_id', 'sentiment', 'review_length', 'word_count', 'helpful_vote', 'verified_purchase']].head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the segmented data\n",
    "df.to_csv('../AmazonReviewDatasets/Final/segmentedReviews.csv', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['rating', 'review_title', 'parent_asin', 'user_id', 'date',\n",
       "       'helpful_vote', 'verified_purchase', 'main_category', 'item_title',\n",
       "       'price', 'cleaned_review', 'sentiment', 'sentiment_encoded',\n",
       "       'review_length', 'word_count'],\n",
       "      dtype='object')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(df.columns)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}