Keras_Titanic_Preprocess.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# note: need to create the directory '/data/' by yourself\n",
    "url = \"http://biostat.mc.vanderbilt.edu/wiki/pub/Main/DataSets/titanic3.xls\"\n",
    "file_path = \"data/titanic3.xls\"\n",
    "if not os.path.isfile(file_path):\n",
    "    result = urllib.request.urlretrieve(url, file_path)\n",
    "    print('downloaded:', result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "import pandas as pd\n",
    "# will use pandas 'dataframe' to do preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_df = pd.read_excel(file_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>survived</th>\n",
       "      <th>name</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>ticket</th>\n",
       "      <th>fare</th>\n",
       "      <th>cabin</th>\n",
       "      <th>embarked</th>\n",
       "      <th>boat</th>\n",
       "      <th>body</th>\n",
       "      <th>home.dest</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Allen, Miss. Elisabeth Walton</td>\n",
       "      <td>female</td>\n",
       "      <td>29.0000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24160</td>\n",
       "      <td>211.3375</td>\n",
       "      <td>B5</td>\n",
       "      <td>S</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>St Louis, MO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Allison, Master. Hudson Trevor</td>\n",
       "      <td>male</td>\n",
       "      <td>0.9167</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>113781</td>\n",
       "      <td>151.5500</td>\n",
       "      <td>C22 C26</td>\n",
       "      <td>S</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Montreal, PQ / Chesterville, ON</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pclass  survived                            name     sex      age  sibsp  \\\n",
       "0       1         1   Allen, Miss. Elisabeth Walton  female  29.0000      0   \n",
       "1       1         1  Allison, Master. Hudson Trevor    male   0.9167      1   \n",
       "\n",
       "   parch  ticket      fare    cabin embarked boat  body  \\\n",
       "0      0   24160  211.3375       B5        S    2   NaN   \n",
       "1      2  113781  151.5500  C22 C26        S   11   NaN   \n",
       "\n",
       "                         home.dest  \n",
       "0                     St Louis, MO  \n",
       "1  Montreal, PQ / Chesterville, ON  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['survived', 'name', 'pclass', 'sex', 'age', 'sibsp', 'parch', 'fare', 'embarked']\n",
    "all_df = all_df[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>survived</th>\n",
       "      <th>name</th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Allen, Miss. Elisabeth Walton</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>29.0000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>211.3375</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Allison, Master. Hudson Trevor</td>\n",
       "      <td>1</td>\n",
       "      <td>male</td>\n",
       "      <td>0.9167</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>151.5500</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   survived                            name  pclass     sex      age  sibsp  \\\n",
       "0         1   Allen, Miss. Elisabeth Walton       1  female  29.0000      0   \n",
       "1         1  Allison, Master. Hudson Trevor       1    male   0.9167      1   \n",
       "\n",
       "   parch      fare embarked  \n",
       "0      0  211.3375        S  \n",
       "1      2  151.5500        S  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = all_df.drop(['name'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "survived      0\n",
       "name          0\n",
       "pclass        0\n",
       "sex           0\n",
       "age         263\n",
       "sibsp         0\n",
       "parch         0\n",
       "fare          1\n",
       "embarked      2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_mean = df['age'].mean()\n",
    "df['age'] = df['age'].fillna(age_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "fare_mean = df['fare'].mean()\n",
    "df['fare'] = df['fare'].fillna(fare_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['sex'] = df['sex'].map( {'female':0, 'male':1} ).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_one_hot_df = pd.get_dummies( data=df, columns=[\"embarked\"] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>survived</th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked_C</th>\n",
       "      <th>embarked_Q</th>\n",
       "      <th>embarked_S</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>29.0000</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>211.3375</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.9167</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>151.5500</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   survived  pclass  sex      age  sibsp  parch      fare  embarked_C  \\\n",
       "0         1       1    0  29.0000      0      0  211.3375           0   \n",
       "1         1       1    1   0.9167      1      2  151.5500           0   \n",
       "\n",
       "   embarked_Q  embarked_S  \n",
       "0           0           1  \n",
       "1           0           1  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_one_hot_df [:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from dataframe to array\n",
    "data_array = x_one_hot_df.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1309, 10)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_array.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  1.    ,   1.    ,   0.    ,  29.    ,   0.    ,   0.    ,\n",
       "        211.3375,   0.    ,   0.    ,   1.    ],\n",
       "       [  1.    ,   1.    ,   1.    ,   0.9167,   1.    ,   2.    ,\n",
       "        151.55  ,   0.    ,   0.    ,   1.    ]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_array[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 1.])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Label\n",
    "label = data_array[: , 0]\n",
    "label[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  1.    ,   0.    ,  29.    ,   0.    ,   0.    , 211.3375,\n",
       "          0.    ,   0.    ,   1.    ],\n",
       "       [  1.    ,   1.    ,   0.9167,   1.    ,   2.    , 151.55  ,\n",
       "          0.    ,   0.    ,   1.    ]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Feature\n",
    "feature = data_array[: ,1:]\n",
    "feature[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 0.        , 0.36116884, 0.        , 0.        ,\n",
       "        0.41250333, 0.        , 0.        , 1.        ],\n",
       "       [0.        , 1.        , 0.00939458, 0.125     , 0.22222222,\n",
       "        0.2958059 , 0.        , 0.        , 1.        ]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use sklearn for standarization \n",
    "from sklearn import preprocessing\n",
    "minmax_scale = preprocessing.MinMaxScaler( feature_range=(0,1) )\n",
    "scaled_features = minmax_scale.fit_transform( feature )\n",
    "scaled_features[:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total: 1309\n",
      "train: 1034\n",
      "test: 275\n"
     ]
    }
   ],
   "source": [
    "# devide into train and test dataframes\n",
    "msk = numpy.random.rand( len(all_df) ) < 0.8\n",
    "train_df = all_df[msk]\n",
    "test_df = all_df[~msk]\n",
    "\n",
    "print('total:', len(all_df))\n",
    "print('train:', len(train_df))\n",
    "print('test:', len(test_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the preprocess function  \n",
    "def preprocess_data(raw_df):\n",
    "    df = raw_df.drop( ['name'], axis=1 )\n",
    "    age_mean = df['age'].mean()\n",
    "    df['age'] = df['age'].fillna(age_mean)\n",
    "    fare_mean = df['fare'].mean()\n",
    "    df['fare'] = df['fare'].fillna(fare_mean)\n",
    "    df['sex'] = df['sex'].map( {'female':0, 'male':1} ).astype(int)\n",
    "    x_one_hot_df = pd.get_dummies( data=df, columns=[\"embarked\" ])\n",
    "    \n",
    "    nd_array = x_one_hot_df.values\n",
    "    feature = nd_array[:, 1:]\n",
    "    label = nd_array[:, 0]\n",
    "    \n",
    "    minmax_scale = preprocessing.MinMaxScaler( feature_range=(0,1) )\n",
    "    scaled_features = minmax_scale.fit_transform( feature )\n",
    "    \n",
    "    return scaled_features,label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features, train_label = preprocess_data( train_df )\n",
    "test_features, test_label = preprocess_data( test_df )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.        , 1.        , 0.00939458, 0.125     , 0.33333333,\n",
       "        0.2958059 , 0.        , 0.        , 1.        ],\n",
       "       [0.        , 0.        , 0.0229641 , 0.125     , 0.33333333,\n",
       "        0.2958059 , 0.        , 0.        , 1.        ],\n",
       "       [0.        , 1.        , 0.37369494, 0.125     , 0.33333333,\n",
       "        0.2958059 , 0.        , 0.        , 1.        ],\n",
       "       [0.        , 0.        , 0.31106443, 0.125     , 0.33333333,\n",
       "        0.2958059 , 0.        , 0.        , 1.        ],\n",
       "       [0.        , 1.        , 0.59916476, 0.        , 0.        ,\n",
       "        0.05182215, 0.        , 0.        , 1.        ]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_features[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1., 0., 0., 0., 1.])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_label[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}