ch1_data_preprocessing_tools.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ThatNinjaGuy/Machine-learning-A-Z-Course/blob/develop/ch1_data_preprocessing_tools.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "37puETfgRzzg"
      },
      "source": [
        "# Data Preprocessing Tools"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EoRP98MpR-qj"
      },
      "source": [
        "## Importing the libraries"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd"
      ],
      "metadata": {
        "id": "t6F0rPY3C12K"
      },
      "execution_count": 61,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RopL7tUZSQkT"
      },
      "source": [
        "## Importing the dataset"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = pd.read_csv('Data.csv')\n",
        "X = dataset.iloc[:, :-1].values\n",
        "Y = dataset.iloc[:, -1].values"
      ],
      "metadata": {
        "id": "RIQVpnUkDxV3"
      },
      "execution_count": 62,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(X)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aOm4BDHUF4pK",
        "outputId": "80886f2a-e501-47f0-980a-50902c9e6310"
      },
      "execution_count": 63,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['France' 44.0 72000.0]\n",
            " ['Spain' 27.0 48000.0]\n",
            " ['Germany' 30.0 54000.0]\n",
            " ['Spain' 38.0 61000.0]\n",
            " ['Germany' 40.0 nan]\n",
            " ['France' 35.0 58000.0]\n",
            " ['Spain' nan 52000.0]\n",
            " ['France' 48.0 79000.0]\n",
            " ['Germany' 50.0 83000.0]\n",
            " ['France' 37.0 67000.0]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(Y)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1UWXTNbrGZ2r",
        "outputId": "b04c5d7f-e862-4bfc-fa56-ed84a215c0d5"
      },
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['No' 'Yes' 'No' 'No' 'Yes' 'Yes' 'No' 'Yes' 'No' 'Yes']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nhfKXNxlSabC"
      },
      "source": [
        "## Taking care of missing data"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.impute import SimpleImputer\n",
        "imputer = SimpleImputer(missing_values=np.nan, strategy='mean')\n",
        "imputer.fit(X[:, 1:3])\n",
        "X[:, 1:3] = imputer.transform(X[:, 1:3])"
      ],
      "metadata": {
        "id": "YcQEwcv3G_dn"
      },
      "execution_count": 65,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(X)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AH3G-VgzH_JJ",
        "outputId": "60b728e6-cd70-4010-8845-f26fce5bf15a"
      },
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['France' 44.0 72000.0]\n",
            " ['Spain' 27.0 48000.0]\n",
            " ['Germany' 30.0 54000.0]\n",
            " ['Spain' 38.0 61000.0]\n",
            " ['Germany' 40.0 63777.77777777778]\n",
            " ['France' 35.0 58000.0]\n",
            " ['Spain' 38.77777777777778 52000.0]\n",
            " ['France' 48.0 79000.0]\n",
            " ['Germany' 50.0 83000.0]\n",
            " ['France' 37.0 67000.0]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CriG6VzVSjcK"
      },
      "source": [
        "## Encoding categorical data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AhSpdQWeSsFh"
      },
      "source": [
        "### Encoding the Independent Variable"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.compose import ColumnTransformer\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [0])],remainder='passthrough')\n",
        "X = np.array(ct.fit_transform(X))\n",
        "# Forcing conversion to np to prepare for future array structure requirement\n",
        "# X = ct.fit_transform(X)"
      ],
      "metadata": {
        "id": "yNWAa4oJKeXI"
      },
      "execution_count": 67,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(X)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eF8JRgL8Ljhu",
        "outputId": "1fd4b830-5048-4b58-d52b-7692ed407ff8"
      },
      "execution_count": 68,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[1.0 0.0 0.0 44.0 72000.0]\n",
            " [0.0 0.0 1.0 27.0 48000.0]\n",
            " [0.0 1.0 0.0 30.0 54000.0]\n",
            " [0.0 0.0 1.0 38.0 61000.0]\n",
            " [0.0 1.0 0.0 40.0 63777.77777777778]\n",
            " [1.0 0.0 0.0 35.0 58000.0]\n",
            " [0.0 0.0 1.0 38.77777777777778 52000.0]\n",
            " [1.0 0.0 0.0 48.0 79000.0]\n",
            " [0.0 1.0 0.0 50.0 83000.0]\n",
            " [1.0 0.0 0.0 37.0 67000.0]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DXh8oVSITIc6"
      },
      "source": [
        "### Encoding the Dependent Variable"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.preprocessing import LabelEncoder\n",
        "le = LabelEncoder()\n",
        "Y = le.fit_transform(Y)"
      ],
      "metadata": {
        "id": "cUzfA7M5MwV4"
      },
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(Y)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gMxjGis-NGic",
        "outputId": "20e60878-8527-41d2-b8e1-fd8b5e1cb93a"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0 1 0 0 1 1 0 1 0 1]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qb_vcgm3qZKW"
      },
      "source": [
        "## Splitting the dataset into the Training set and Test set"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "# Dataset splitting should be done before feature scaling.\n",
        "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 1)"
      ],
      "metadata": {
        "id": "zwW2OUJENgGn"
      },
      "execution_count": 71,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(X_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eRAG_mdIPBBK",
        "outputId": "f722a953-2649-494d-c8cd-72fc51b01cf9"
      },
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[0.0 0.0 1.0 38.77777777777778 52000.0]\n",
            " [0.0 1.0 0.0 40.0 63777.77777777778]\n",
            " [1.0 0.0 0.0 44.0 72000.0]\n",
            " [0.0 0.0 1.0 38.0 61000.0]\n",
            " [0.0 0.0 1.0 27.0 48000.0]\n",
            " [1.0 0.0 0.0 48.0 79000.0]\n",
            " [0.0 1.0 0.0 50.0 83000.0]\n",
            " [1.0 0.0 0.0 35.0 58000.0]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(Y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gXxlHnhSPFPp",
        "outputId": "b934bf02-1611-4431-87d9-cea83ca80b68"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0 1 0 0 1 1 0 1]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(X_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "t-A0u5XoPI0X",
        "outputId": "2f888569-78b4-4658-cb19-07fc0b19376f"
      },
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[0.0 1.0 0.0 30.0 54000.0]\n",
            " [1.0 0.0 0.0 37.0 67000.0]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(Y_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5Wp1bA6LPQGx",
        "outputId": "9400112a-4e2e-4018-e7d5-ec16ce192b3e"
      },
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0 1]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TpGqbS4TqkIR"
      },
      "source": [
        "## Feature Scaling"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.preprocessing import StandardScaler\n",
        "sc = StandardScaler()\n",
        "# Feature scaling should not be applied to the dummy data that has been generated from the non-numerical columns.\n",
        "# Atleast in this flow as the dummy data for this code involves the dumy data being generated from the countries column\n",
        "X_train[:, 3:] = sc.fit_transform(X_train[:, 3:])\n",
        "# Fit calculates the mean and standard deviation but doesn't transform the data\n",
        "# The standard scaler object stores these mean and deviation by fitting it\n",
        "X_test[:, 3:] = sc.transform(X_test[:, 3:])\n"
      ],
      "metadata": {
        "id": "kzVr0_DmQXBY"
      },
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(X_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eg2c39pCT19k",
        "outputId": "e2d06120-246b-41cb-e31f-3303713f2e1f"
      },
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[0.0 0.0 1.0 -0.19159184384578545 -1.0781259408412425]\n",
            " [0.0 1.0 0.0 -0.014117293757057777 -0.07013167641635372]\n",
            " [1.0 0.0 0.0 0.566708506533324 0.633562432710455]\n",
            " [0.0 0.0 1.0 -0.30453019390224867 -0.30786617274297867]\n",
            " [0.0 0.0 1.0 -1.9018011447007988 -1.420463615551582]\n",
            " [1.0 0.0 0.0 1.1475343068237058 1.232653363453549]\n",
            " [0.0 1.0 0.0 1.4379472069688968 1.5749910381638885]\n",
            " [1.0 0.0 0.0 -0.7401495441200351 -0.5646194287757332]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(X_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7T6MBWsvT3J_",
        "outputId": "8d8cf545-ecaa-4300-b1f2-12c4a85e2655"
      },
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[0.0 1.0 0.0 -1.4661817944830124 -0.9069571034860727]\n",
            " [1.0 0.0 0.0 -0.44973664397484414 0.2056403393225306]]\n"
          ]
        }
      ]
    }
  ]
}