thesis/categorical_features.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from IPython.display import display\n",
    "from scipy.stats import chi2_contingency\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "from utils import load_train_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_filename = \"application_train_categorical.csv\"\n",
    "path_to_data = \"../data/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load train categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_application_train_categorical = load_train_test(\n",
    "    filename=train_filename,\n",
    "    path_to_train_folder=path_to_data,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### contingency matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>F</th>\n",
       "      <td>188278</td>\n",
       "      <td>14170</td>\n",
       "      <td>202448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>M</th>\n",
       "      <td>94404</td>\n",
       "      <td>10655</td>\n",
       "      <td>105059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XNA</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>282686</td>\n",
       "      <td>24825</td>\n",
       "      <td>307511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET            0      1     All\n",
       "CODE_GENDER                       \n",
       "F            188278  14170  202448\n",
       "M             94404  10655  105059\n",
       "XNA               4      0       4\n",
       "All          282686  24825  307511"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.0\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NAME_TYPE_SUITE</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Children</th>\n",
       "      <td>3026</td>\n",
       "      <td>241</td>\n",
       "      <td>3267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Family</th>\n",
       "      <td>37140</td>\n",
       "      <td>3009</td>\n",
       "      <td>40149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Group of people</th>\n",
       "      <td>248</td>\n",
       "      <td>23</td>\n",
       "      <td>271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Other_A</th>\n",
       "      <td>790</td>\n",
       "      <td>76</td>\n",
       "      <td>866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Other_B</th>\n",
       "      <td>1596</td>\n",
       "      <td>174</td>\n",
       "      <td>1770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Spouse, partner</th>\n",
       "      <td>10475</td>\n",
       "      <td>895</td>\n",
       "      <td>11370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unaccompanied</th>\n",
       "      <td>228189</td>\n",
       "      <td>20337</td>\n",
       "      <td>248526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>281464</td>\n",
       "      <td>24755</td>\n",
       "      <td>306219</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                0      1     All\n",
       "NAME_TYPE_SUITE                       \n",
       "Children           3026    241    3267\n",
       "Family            37140   3009   40149\n",
       "Group of people     248     23     271\n",
       "Other_A             790     76     866\n",
       "Other_B            1596    174    1770\n",
       "Spouse, partner   10475    895   11370\n",
       "Unaccompanied    228189  20337  248526\n",
       "All              281464  24755  306219"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.003\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NAME_INCOME_TYPE</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Businessman</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Commercial associate</th>\n",
       "      <td>66257</td>\n",
       "      <td>5360</td>\n",
       "      <td>71617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Maternity leave</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pensioner</th>\n",
       "      <td>52380</td>\n",
       "      <td>2982</td>\n",
       "      <td>55362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>State servant</th>\n",
       "      <td>20454</td>\n",
       "      <td>1249</td>\n",
       "      <td>21703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Student</th>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unemployed</th>\n",
       "      <td>14</td>\n",
       "      <td>8</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Working</th>\n",
       "      <td>143550</td>\n",
       "      <td>15224</td>\n",
       "      <td>158774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>282686</td>\n",
       "      <td>24825</td>\n",
       "      <td>307511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                     0      1     All\n",
       "NAME_INCOME_TYPE                           \n",
       "Businessman               10      0      10\n",
       "Commercial associate   66257   5360   71617\n",
       "Maternity leave            3      2       5\n",
       "Pensioner              52380   2982   55362\n",
       "State servant          20454   1249   21703\n",
       "Student                   18      0      18\n",
       "Unemployed                14      8      22\n",
       "Working               143550  15224  158774\n",
       "All                   282686  24825  307511"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.0\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NAME_EDUCATION_TYPE</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Academic degree</th>\n",
       "      <td>161</td>\n",
       "      <td>3</td>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Higher education</th>\n",
       "      <td>70854</td>\n",
       "      <td>4009</td>\n",
       "      <td>74863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Incomplete higher</th>\n",
       "      <td>9405</td>\n",
       "      <td>872</td>\n",
       "      <td>10277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lower secondary</th>\n",
       "      <td>3399</td>\n",
       "      <td>417</td>\n",
       "      <td>3816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Secondary / secondary special</th>\n",
       "      <td>198867</td>\n",
       "      <td>19524</td>\n",
       "      <td>218391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>282686</td>\n",
       "      <td>24825</td>\n",
       "      <td>307511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                              0      1     All\n",
       "NAME_EDUCATION_TYPE                                 \n",
       "Academic degree                   161      3     164\n",
       "Higher education                70854   4009   74863\n",
       "Incomplete higher                9405    872   10277\n",
       "Lower secondary                  3399    417    3816\n",
       "Secondary / secondary special  198867  19524  218391\n",
       "All                            282686  24825  307511"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.0\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NAME_FAMILY_STATUS</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Civil marriage</th>\n",
       "      <td>26814</td>\n",
       "      <td>2961</td>\n",
       "      <td>29775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Married</th>\n",
       "      <td>181582</td>\n",
       "      <td>14850</td>\n",
       "      <td>196432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Separated</th>\n",
       "      <td>18150</td>\n",
       "      <td>1620</td>\n",
       "      <td>19770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Single / not married</th>\n",
       "      <td>40987</td>\n",
       "      <td>4457</td>\n",
       "      <td>45444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unknown</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Widow</th>\n",
       "      <td>15151</td>\n",
       "      <td>937</td>\n",
       "      <td>16088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>282686</td>\n",
       "      <td>24825</td>\n",
       "      <td>307511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                     0      1     All\n",
       "NAME_FAMILY_STATUS                         \n",
       "Civil marriage         26814   2961   29775\n",
       "Married               181582  14850  196432\n",
       "Separated              18150   1620   19770\n",
       "Single / not married   40987   4457   45444\n",
       "Unknown                    2      0       2\n",
       "Widow                  15151    937   16088\n",
       "All                   282686  24825  307511"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.0\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NAME_HOUSING_TYPE</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Co-op apartment</th>\n",
       "      <td>1033</td>\n",
       "      <td>89</td>\n",
       "      <td>1122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>House / apartment</th>\n",
       "      <td>251596</td>\n",
       "      <td>21272</td>\n",
       "      <td>272868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Municipal apartment</th>\n",
       "      <td>10228</td>\n",
       "      <td>955</td>\n",
       "      <td>11183</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Office apartment</th>\n",
       "      <td>2445</td>\n",
       "      <td>172</td>\n",
       "      <td>2617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Rented apartment</th>\n",
       "      <td>4280</td>\n",
       "      <td>601</td>\n",
       "      <td>4881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>With parents</th>\n",
       "      <td>13104</td>\n",
       "      <td>1736</td>\n",
       "      <td>14840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>282686</td>\n",
       "      <td>24825</td>\n",
       "      <td>307511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                    0      1     All\n",
       "NAME_HOUSING_TYPE                         \n",
       "Co-op apartment        1033     89    1122\n",
       "House / apartment    251596  21272  272868\n",
       "Municipal apartment   10228    955   11183\n",
       "Office apartment       2445    172    2617\n",
       "Rented apartment       4280    601    4881\n",
       "With parents          13104   1736   14840\n",
       "All                  282686  24825  307511"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.0\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>OCCUPATION_TYPE</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Accountants</th>\n",
       "      <td>9339</td>\n",
       "      <td>474</td>\n",
       "      <td>9813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cleaning staff</th>\n",
       "      <td>4206</td>\n",
       "      <td>447</td>\n",
       "      <td>4653</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cooking staff</th>\n",
       "      <td>5325</td>\n",
       "      <td>621</td>\n",
       "      <td>5946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Core staff</th>\n",
       "      <td>25832</td>\n",
       "      <td>1738</td>\n",
       "      <td>27570</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Drivers</th>\n",
       "      <td>16496</td>\n",
       "      <td>2107</td>\n",
       "      <td>18603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HR staff</th>\n",
       "      <td>527</td>\n",
       "      <td>36</td>\n",
       "      <td>563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>High skill tech staff</th>\n",
       "      <td>10679</td>\n",
       "      <td>701</td>\n",
       "      <td>11380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IT staff</th>\n",
       "      <td>492</td>\n",
       "      <td>34</td>\n",
       "      <td>526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Laborers</th>\n",
       "      <td>49348</td>\n",
       "      <td>5838</td>\n",
       "      <td>55186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Low-skill Laborers</th>\n",
       "      <td>1734</td>\n",
       "      <td>359</td>\n",
       "      <td>2093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Managers</th>\n",
       "      <td>20043</td>\n",
       "      <td>1328</td>\n",
       "      <td>21371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Medicine staff</th>\n",
       "      <td>7965</td>\n",
       "      <td>572</td>\n",
       "      <td>8537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Private service staff</th>\n",
       "      <td>2477</td>\n",
       "      <td>175</td>\n",
       "      <td>2652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Realty agents</th>\n",
       "      <td>692</td>\n",
       "      <td>59</td>\n",
       "      <td>751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sales staff</th>\n",
       "      <td>29010</td>\n",
       "      <td>3092</td>\n",
       "      <td>32102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Secretaries</th>\n",
       "      <td>1213</td>\n",
       "      <td>92</td>\n",
       "      <td>1305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Security staff</th>\n",
       "      <td>5999</td>\n",
       "      <td>722</td>\n",
       "      <td>6721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Waiters/barmen staff</th>\n",
       "      <td>1196</td>\n",
       "      <td>152</td>\n",
       "      <td>1348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>192573</td>\n",
       "      <td>18547</td>\n",
       "      <td>211120</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                      0      1     All\n",
       "OCCUPATION_TYPE                             \n",
       "Accountants              9339    474    9813\n",
       "Cleaning staff           4206    447    4653\n",
       "Cooking staff            5325    621    5946\n",
       "Core staff              25832   1738   27570\n",
       "Drivers                 16496   2107   18603\n",
       "HR staff                  527     36     563\n",
       "High skill tech staff   10679    701   11380\n",
       "IT staff                  492     34     526\n",
       "Laborers                49348   5838   55186\n",
       "Low-skill Laborers       1734    359    2093\n",
       "Managers                20043   1328   21371\n",
       "Medicine staff           7965    572    8537\n",
       "Private service staff    2477    175    2652\n",
       "Realty agents             692     59     751\n",
       "Sales staff             29010   3092   32102\n",
       "Secretaries              1213     92    1305\n",
       "Security staff           5999    722    6721\n",
       "Waiters/barmen staff     1196    152    1348\n",
       "All                    192573  18547  211120"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.0\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>REGION_RATING_CLIENT</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>30645</td>\n",
       "      <td>1552</td>\n",
       "      <td>32197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>209077</td>\n",
       "      <td>17907</td>\n",
       "      <td>226984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>42964</td>\n",
       "      <td>5366</td>\n",
       "      <td>48330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>282686</td>\n",
       "      <td>24825</td>\n",
       "      <td>307511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                     0      1     All\n",
       "REGION_RATING_CLIENT                       \n",
       "1                      30645   1552   32197\n",
       "2                     209077  17907  226984\n",
       "3                      42964   5366   48330\n",
       "All                   282686  24825  307511"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.0\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>REGION_RATING_CLIENT_W_CITY</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>32513</td>\n",
       "      <td>1654</td>\n",
       "      <td>34167</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>211314</td>\n",
       "      <td>18170</td>\n",
       "      <td>229484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>38859</td>\n",
       "      <td>5001</td>\n",
       "      <td>43860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>282686</td>\n",
       "      <td>24825</td>\n",
       "      <td>307511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                            0      1     All\n",
       "REGION_RATING_CLIENT_W_CITY                       \n",
       "1                             32513   1654   34167\n",
       "2                            211314  18170  229484\n",
       "3                             38859   5001   43860\n",
       "All                          282686  24825  307511"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.0\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WEEKDAY_APPR_PROCESS_START</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>FRIDAY</th>\n",
       "      <td>46237</td>\n",
       "      <td>4101</td>\n",
       "      <td>50338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MONDAY</th>\n",
       "      <td>46780</td>\n",
       "      <td>3934</td>\n",
       "      <td>50714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SATURDAY</th>\n",
       "      <td>31182</td>\n",
       "      <td>2670</td>\n",
       "      <td>33852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SUNDAY</th>\n",
       "      <td>14898</td>\n",
       "      <td>1283</td>\n",
       "      <td>16181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>THURSDAY</th>\n",
       "      <td>46493</td>\n",
       "      <td>4098</td>\n",
       "      <td>50591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TUESDAY</th>\n",
       "      <td>49400</td>\n",
       "      <td>4501</td>\n",
       "      <td>53901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WEDNESDAY</th>\n",
       "      <td>47696</td>\n",
       "      <td>4238</td>\n",
       "      <td>51934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>282686</td>\n",
       "      <td>24825</td>\n",
       "      <td>307511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                           0      1     All\n",
       "WEEKDAY_APPR_PROCESS_START                       \n",
       "FRIDAY                       46237   4101   50338\n",
       "MONDAY                       46780   3934   50714\n",
       "SATURDAY                     31182   2670   33852\n",
       "SUNDAY                       14898   1283   16181\n",
       "THURSDAY                     46493   4098   50591\n",
       "TUESDAY                      49400   4501   53901\n",
       "WEDNESDAY                    47696   4238   51934\n",
       "All                         282686  24825  307511"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.352\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ORGANIZATION_TYPE</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Advertising</th>\n",
       "      <td>394</td>\n",
       "      <td>35</td>\n",
       "      <td>429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Agriculture</th>\n",
       "      <td>2197</td>\n",
       "      <td>257</td>\n",
       "      <td>2454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bank</th>\n",
       "      <td>2377</td>\n",
       "      <td>130</td>\n",
       "      <td>2507</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Business Entity Type 1</th>\n",
       "      <td>5497</td>\n",
       "      <td>487</td>\n",
       "      <td>5984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Business Entity Type 2</th>\n",
       "      <td>9653</td>\n",
       "      <td>900</td>\n",
       "      <td>10553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Business Entity Type 3</th>\n",
       "      <td>61669</td>\n",
       "      <td>6323</td>\n",
       "      <td>67992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cleaning</th>\n",
       "      <td>231</td>\n",
       "      <td>29</td>\n",
       "      <td>260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Construction</th>\n",
       "      <td>5936</td>\n",
       "      <td>785</td>\n",
       "      <td>6721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Culture</th>\n",
       "      <td>358</td>\n",
       "      <td>21</td>\n",
       "      <td>379</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Electricity</th>\n",
       "      <td>887</td>\n",
       "      <td>63</td>\n",
       "      <td>950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Emergency</th>\n",
       "      <td>520</td>\n",
       "      <td>40</td>\n",
       "      <td>560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Government</th>\n",
       "      <td>9678</td>\n",
       "      <td>726</td>\n",
       "      <td>10404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Hotel</th>\n",
       "      <td>904</td>\n",
       "      <td>62</td>\n",
       "      <td>966</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Housing</th>\n",
       "      <td>2723</td>\n",
       "      <td>235</td>\n",
       "      <td>2958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 1</th>\n",
       "      <td>924</td>\n",
       "      <td>115</td>\n",
       "      <td>1039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 10</th>\n",
       "      <td>102</td>\n",
       "      <td>7</td>\n",
       "      <td>109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 11</th>\n",
       "      <td>2470</td>\n",
       "      <td>234</td>\n",
       "      <td>2704</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 12</th>\n",
       "      <td>355</td>\n",
       "      <td>14</td>\n",
       "      <td>369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 13</th>\n",
       "      <td>58</td>\n",
       "      <td>9</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 2</th>\n",
       "      <td>425</td>\n",
       "      <td>33</td>\n",
       "      <td>458</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 3</th>\n",
       "      <td>2930</td>\n",
       "      <td>348</td>\n",
       "      <td>3278</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 4</th>\n",
       "      <td>788</td>\n",
       "      <td>89</td>\n",
       "      <td>877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 5</th>\n",
       "      <td>558</td>\n",
       "      <td>41</td>\n",
       "      <td>599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 6</th>\n",
       "      <td>104</td>\n",
       "      <td>8</td>\n",
       "      <td>112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 7</th>\n",
       "      <td>1202</td>\n",
       "      <td>105</td>\n",
       "      <td>1307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 8</th>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Industry: type 9</th>\n",
       "      <td>3143</td>\n",
       "      <td>225</td>\n",
       "      <td>3368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Insurance</th>\n",
       "      <td>563</td>\n",
       "      <td>34</td>\n",
       "      <td>597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kindergarten</th>\n",
       "      <td>6396</td>\n",
       "      <td>484</td>\n",
       "      <td>6880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Legal Services</th>\n",
       "      <td>281</td>\n",
       "      <td>24</td>\n",
       "      <td>305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Medicine</th>\n",
       "      <td>10456</td>\n",
       "      <td>737</td>\n",
       "      <td>11193</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Military</th>\n",
       "      <td>2499</td>\n",
       "      <td>135</td>\n",
       "      <td>2634</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mobile</th>\n",
       "      <td>288</td>\n",
       "      <td>29</td>\n",
       "      <td>317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Other</th>\n",
       "      <td>15408</td>\n",
       "      <td>1275</td>\n",
       "      <td>16683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Police</th>\n",
       "      <td>2224</td>\n",
       "      <td>117</td>\n",
       "      <td>2341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Postal</th>\n",
       "      <td>1975</td>\n",
       "      <td>182</td>\n",
       "      <td>2157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Realtor</th>\n",
       "      <td>354</td>\n",
       "      <td>42</td>\n",
       "      <td>396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Religion</th>\n",
       "      <td>80</td>\n",
       "      <td>5</td>\n",
       "      <td>85</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Restaurant</th>\n",
       "      <td>1599</td>\n",
       "      <td>212</td>\n",
       "      <td>1811</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>School</th>\n",
       "      <td>8367</td>\n",
       "      <td>526</td>\n",
       "      <td>8893</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Security</th>\n",
       "      <td>2923</td>\n",
       "      <td>324</td>\n",
       "      <td>3247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Security Ministries</th>\n",
       "      <td>1878</td>\n",
       "      <td>96</td>\n",
       "      <td>1974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Self-employed</th>\n",
       "      <td>34504</td>\n",
       "      <td>3908</td>\n",
       "      <td>38412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Services</th>\n",
       "      <td>1471</td>\n",
       "      <td>104</td>\n",
       "      <td>1575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Telecom</th>\n",
       "      <td>533</td>\n",
       "      <td>44</td>\n",
       "      <td>577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Trade: type 1</th>\n",
       "      <td>317</td>\n",
       "      <td>31</td>\n",
       "      <td>348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Trade: type 2</th>\n",
       "      <td>1767</td>\n",
       "      <td>133</td>\n",
       "      <td>1900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Trade: type 3</th>\n",
       "      <td>3131</td>\n",
       "      <td>361</td>\n",
       "      <td>3492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Trade: type 4</th>\n",
       "      <td>62</td>\n",
       "      <td>2</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Trade: type 5</th>\n",
       "      <td>46</td>\n",
       "      <td>3</td>\n",
       "      <td>49</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Trade: type 6</th>\n",
       "      <td>602</td>\n",
       "      <td>29</td>\n",
       "      <td>631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Trade: type 7</th>\n",
       "      <td>7091</td>\n",
       "      <td>740</td>\n",
       "      <td>7831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transport: type 1</th>\n",
       "      <td>192</td>\n",
       "      <td>9</td>\n",
       "      <td>201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transport: type 2</th>\n",
       "      <td>2032</td>\n",
       "      <td>172</td>\n",
       "      <td>2204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transport: type 3</th>\n",
       "      <td>1000</td>\n",
       "      <td>187</td>\n",
       "      <td>1187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transport: type 4</th>\n",
       "      <td>4897</td>\n",
       "      <td>501</td>\n",
       "      <td>5398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>University</th>\n",
       "      <td>1262</td>\n",
       "      <td>65</td>\n",
       "      <td>1327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XNA</th>\n",
       "      <td>52384</td>\n",
       "      <td>2990</td>\n",
       "      <td>55374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>282686</td>\n",
       "      <td>24825</td>\n",
       "      <td>307511</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                       0      1     All\n",
       "ORGANIZATION_TYPE                            \n",
       "Advertising                394     35     429\n",
       "Agriculture               2197    257    2454\n",
       "Bank                      2377    130    2507\n",
       "Business Entity Type 1    5497    487    5984\n",
       "Business Entity Type 2    9653    900   10553\n",
       "Business Entity Type 3   61669   6323   67992\n",
       "Cleaning                   231     29     260\n",
       "Construction              5936    785    6721\n",
       "Culture                    358     21     379\n",
       "Electricity                887     63     950\n",
       "Emergency                  520     40     560\n",
       "Government                9678    726   10404\n",
       "Hotel                      904     62     966\n",
       "Housing                   2723    235    2958\n",
       "Industry: type 1           924    115    1039\n",
       "Industry: type 10          102      7     109\n",
       "Industry: type 11         2470    234    2704\n",
       "Industry: type 12          355     14     369\n",
       "Industry: type 13           58      9      67\n",
       "Industry: type 2           425     33     458\n",
       "Industry: type 3          2930    348    3278\n",
       "Industry: type 4           788     89     877\n",
       "Industry: type 5           558     41     599\n",
       "Industry: type 6           104      8     112\n",
       "Industry: type 7          1202    105    1307\n",
       "Industry: type 8            21      3      24\n",
       "Industry: type 9          3143    225    3368\n",
       "Insurance                  563     34     597\n",
       "Kindergarten              6396    484    6880\n",
       "Legal Services             281     24     305\n",
       "Medicine                 10456    737   11193\n",
       "Military                  2499    135    2634\n",
       "Mobile                     288     29     317\n",
       "Other                    15408   1275   16683\n",
       "Police                    2224    117    2341\n",
       "Postal                    1975    182    2157\n",
       "Realtor                    354     42     396\n",
       "Religion                    80      5      85\n",
       "Restaurant                1599    212    1811\n",
       "School                    8367    526    8893\n",
       "Security                  2923    324    3247\n",
       "Security Ministries       1878     96    1974\n",
       "Self-employed            34504   3908   38412\n",
       "Services                  1471    104    1575\n",
       "Telecom                    533     44     577\n",
       "Trade: type 1              317     31     348\n",
       "Trade: type 2             1767    133    1900\n",
       "Trade: type 3             3131    361    3492\n",
       "Trade: type 4               62      2      64\n",
       "Trade: type 5               46      3      49\n",
       "Trade: type 6              602     29     631\n",
       "Trade: type 7             7091    740    7831\n",
       "Transport: type 1          192      9     201\n",
       "Transport: type 2         2032    172    2204\n",
       "Transport: type 3         1000    187    1187\n",
       "Transport: type 4         4897    501    5398\n",
       "University                1262     65    1327\n",
       "XNA                      52384   2990   55374\n",
       "All                     282686  24825  307511"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.0\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FONDKAPREMONT_MODE</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>not specified</th>\n",
       "      <td>5258</td>\n",
       "      <td>429</td>\n",
       "      <td>5687</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>org spec account</th>\n",
       "      <td>5292</td>\n",
       "      <td>327</td>\n",
       "      <td>5619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reg oper account</th>\n",
       "      <td>68678</td>\n",
       "      <td>5152</td>\n",
       "      <td>73830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reg oper spec account</th>\n",
       "      <td>11288</td>\n",
       "      <td>792</td>\n",
       "      <td>12080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>90516</td>\n",
       "      <td>6700</td>\n",
       "      <td>97216</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                     0     1    All\n",
       "FONDKAPREMONT_MODE                       \n",
       "not specified           5258   429   5687\n",
       "org spec account        5292   327   5619\n",
       "reg oper account       68678  5152  73830\n",
       "reg oper spec account  11288   792  12080\n",
       "All                    90516  6700  97216"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.032\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HOUSETYPE_MODE</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>block of flats</th>\n",
       "      <td>140053</td>\n",
       "      <td>10450</td>\n",
       "      <td>150503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>specific housing</th>\n",
       "      <td>1347</td>\n",
       "      <td>152</td>\n",
       "      <td>1499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>terraced house</th>\n",
       "      <td>1109</td>\n",
       "      <td>103</td>\n",
       "      <td>1212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>142509</td>\n",
       "      <td>10705</td>\n",
       "      <td>153214</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                 0      1     All\n",
       "HOUSETYPE_MODE                         \n",
       "block of flats    140053  10450  150503\n",
       "specific housing    1347    152    1499\n",
       "terraced house      1109    103    1212\n",
       "All               142509  10705  153214"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.0\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>TARGET</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>All</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>WALLSMATERIAL_MODE</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Block</th>\n",
       "      <td>8603</td>\n",
       "      <td>650</td>\n",
       "      <td>9253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mixed</th>\n",
       "      <td>2123</td>\n",
       "      <td>173</td>\n",
       "      <td>2296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Monolithic</th>\n",
       "      <td>1695</td>\n",
       "      <td>84</td>\n",
       "      <td>1779</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Others</th>\n",
       "      <td>1490</td>\n",
       "      <td>135</td>\n",
       "      <td>1625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Panel</th>\n",
       "      <td>61848</td>\n",
       "      <td>4192</td>\n",
       "      <td>66040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Stone, brick</th>\n",
       "      <td>60015</td>\n",
       "      <td>4800</td>\n",
       "      <td>64815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Wooden</th>\n",
       "      <td>4842</td>\n",
       "      <td>520</td>\n",
       "      <td>5362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All</th>\n",
       "      <td>140616</td>\n",
       "      <td>10554</td>\n",
       "      <td>151170</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "TARGET                   0      1     All\n",
       "WALLSMATERIAL_MODE                       \n",
       "Block                 8603    650    9253\n",
       "Mixed                 2123    173    2296\n",
       "Monolithic            1695     84    1779\n",
       "Others                1490    135    1625\n",
       "Panel                61848   4192   66040\n",
       "Stone, brick         60015   4800   64815\n",
       "Wooden                4842    520    5362\n",
       "All                 140616  10554  151170"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value: 0.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# chi-square test of independence of variables in a contingency matrix\n",
    "for column in df_application_train_categorical.columns[1:]:  # exclude target\n",
    "    \n",
    "    feature = df_application_train_categorical[column]\n",
    "    target = df_application_train_categorical['TARGET']\n",
    "    \n",
    "    target_crosstab = pd.crosstab(\n",
    "        index=feature,\n",
    "        columns=target,\n",
    "        margins=True,\n",
    "    )\n",
    "    display(target_crosstab)\n",
    "    \n",
    "    _, p_value, _, _ = chi2_contingency(target_crosstab)\n",
    "    print(f\"p-value: {round(p_value, 3)}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CODE_GENDER matthews corr: 0.055\n",
      "NAME_TYPE_SUITE matthews corr: -0.003\n",
      "NAME_INCOME_TYPE matthews corr: -0.004\n",
      "NAME_EDUCATION_TYPE matthews corr: -0.026\n",
      "NAME_FAMILY_STATUS matthews corr: -0.018\n",
      "NAME_HOUSING_TYPE matthews corr: -0.014\n",
      "OCCUPATION_TYPE matthews corr: 0.004\n",
      "REGION_RATING_CLIENT matthews corr: -0.014\n",
      "REGION_RATING_CLIENT_W_CITY matthews corr: -0.015\n",
      "WEEKDAY_APPR_PROCESS_START matthews corr: -0.002\n",
      "ORGANIZATION_TYPE matthews corr: 0.001\n",
      "FONDKAPREMONT_MODE matthews corr: -0.001\n",
      "HOUSETYPE_MODE matthews corr: 0.02\n",
      "WALLSMATERIAL_MODE matthews corr: 0.001\n"
     ]
    }
   ],
   "source": [
    "# chi-square test significance matthews_corrcoef\n",
    "for column in df_application_train_categorical.columns[1:]:  # exclude target\n",
    "    \n",
    "    n_samples = sum(~df_application_train_categorical[column].isna())\n",
    "    target = df_application_train_categorical['TARGET']\n",
    "    \n",
    "    if pd.api.types.is_string_dtype(df_application_train_categorical[column]):\n",
    "        feature = LabelEncoder().fit_transform(df_application_train_categorical[column])\n",
    "    else:\n",
    "        feature = df_application_train_categorical[column]\n",
    "    \n",
    "    matthews_coef = matthews_corrcoef(\n",
    "        y_pred=feature,\n",
    "        y_true=target,\n",
    "    )\n",
    "    \n",
    "    print(f\"{column} matthews corr: {round(matthews_coef, 3)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}