AudiobooksPreprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "89a72ad2-1525-4e58-b9ac-b8a8676087b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import preprocessing #standardize all inputs\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6359c7d-0e36-46f3-89e5-954535dd39e0",
   "metadata": {},
   "source": [
    "#### 2) Extract the data from the Audiobooks csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "847f9667-9690-4dd6-b439-1e82a70f42af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of csv file: (14084, 12)\n"
     ]
    }
   ],
   "source": [
    "#Numpy loadtxt() function is used to load the data from a text file and store them in a ndarray.\n",
    "#Each row in the text file must have the same number of values.\n",
    "raw_csv_data = np.loadtxt(\"Audiobooks_data.csv\", delimiter = ',')\n",
    "\n",
    "print(\"Shape of csv file: \" + str(raw_csv_data.shape))\n",
    "\n",
    "#Take all columns excluding ID and targets columns.\n",
    "unscaled_inputs_all = raw_csv_data[:,1:-1]\n",
    "\n",
    "#Take just targets column.\n",
    "targets_all = raw_csv_data[:,-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9aa87b9d-c2e2-47e7-9891-04eea9a6f938",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape of unscaled_input variable: (14084, 10)\n",
      "Shape of targets variable: (14084,)\n"
     ]
    }
   ],
   "source": [
    "print(\"Shape of unscaled_input variable: \" + str(unscaled_inputs_all.shape))\n",
    "print(\"Shape of targets variable: \" + str(targets_all.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "54e9b44d-aa32-4899-bd86-530219ed9caf",
   "metadata": {},
   "source": [
    "#### 3. Balancing the Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6bbc2866-534f-46ca-95d1-8587eddf71ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Target's 0 and 1 is imbalanced form if we sum all targets the result is count of number 1 variable in dataset.\n",
    "count_one_targets = int(np.sum(targets_all))\n",
    "zero_targets_counter = 0\n",
    "indices_to_remove = []\n",
    "\n",
    "#on axis=0 shape is length of the targets_all vector\n",
    "for i in range(targets_all.shape[0]):\n",
    "    if targets_all[i] == 0:\n",
    "        zero_targets_counter += 1\n",
    "        if zero_targets_counter > count_one_targets:\n",
    "            indices_to_remove.append(i)\n",
    "            \n",
    "#np.delete(array, obj to delete, axis) is a method that deletes an object along an axis            \n",
    "unscaled_inputs_equal_priors = np.delete(unscaled_inputs_all, indices_to_remove, axis=0)\n",
    "targets_equal_priors = np.delete(targets_all, indices_to_remove, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38a5d119-aefc-490f-99c9-4a7f85195681",
   "metadata": {},
   "source": [
    "#### 4. Standardize the inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5af018a2-950c-4df7-89ee-df3456d46343",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaled_inputs = preprocessing.scale(unscaled_inputs_equal_priors)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf986587-edb7-4490-aeb6-9ac6b40e6399",
   "metadata": {},
   "source": [
    "#### 5. Shuffle the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "daeadb7a-97ee-455e-886d-d60b9229afa0",
   "metadata": {},
   "outputs": [],
   "source": [
    "shuffled_indices = np.arange(scaled_inputs.shape[0])\n",
    "np.random.shuffle(shuffled_indices)\n",
    "\n",
    "shuffled_inputs = scaled_inputs[shuffled_indices]\n",
    "shuffled_targets = targets_equal_priors[shuffled_indices]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25aa2fc7-c714-42e3-963b-a05cb3bea5af",
   "metadata": {},
   "source": [
    "#### Split the dataset into Train, Validation, Test "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ee30ad11-b88f-40de-9e95-33a13ab588ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 80-10-10\n",
    "samples_count = shuffled_inputs.shape[0]\n",
    "\n",
    "train_samples_count = int(0.8*samples_count)\n",
    "validation_samples_count = int(0.1*samples_count)\n",
    "test_samples_count = samples_count - train_samples_count - validation_samples_count\n",
    "\n",
    "train_inputs = shuffled_inputs[:train_samples_count]\n",
    "train_targets = shuffled_targets[:train_samples_count]\n",
    "\n",
    "validation_inputs = shuffled_inputs[train_samples_count:train_samples_count+validation_samples_count]\n",
    "validation_targets = shuffled_targets[train_samples_count:train_samples_count+validation_samples_count]\n",
    "\n",
    "test_inputs = shuffled_inputs[train_samples_count+validation_samples_count:]\n",
    "test_targets = shuffled_targets[train_samples_count+validation_samples_count:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "666c70c3-de93-4298-b8f5-d0766ffe97c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1787.0 3579 0.4993014808605756\n",
      "221.0 447 0.49440715883668906\n",
      "229.0 448 0.5111607142857143\n"
     ]
    }
   ],
   "source": [
    "print(np.sum(train_targets), train_samples_count, np.sum(train_targets) / train_samples_count)\n",
    "print(np.sum(validation_targets), validation_samples_count, np.sum(validation_targets) / validation_samples_count)\n",
    "print(np.sum(test_targets), test_samples_count, np.sum(test_targets) / test_samples_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b051bdd2-bb78-49f7-a6b4-499b19b14a78",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.savez(\"Audiobooks_data_train\", inputs=train_inputs, targets=train_targets)\n",
    "np.savez(\"Audiobooks_data_validation\", inputs=validation_inputs, targets=validation_targets)\n",
    "np.savez(\"Audiobooks_data_test\", inputs=test_inputs, targets=test_targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9ae45769-13f5-4d42-9de6-7503faa85c75",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}