01_synpop_explore.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp synpop_explore"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# synpop_explore\n",
    "\n",
    "> API details."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install nbdev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import synthpop.categorizer as cat\n",
    "import multiprocessing\n",
    "import synthpop.zone_synthesizer as zs\n",
    "from functools import partial\n",
    "from synthpop.synthesizer import synthesize, enable_logging"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load sample and marginal distribution data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hh_marginal_file = 'data/example/hh_marginals.csv'\n",
    "person_marginal_file = 'data/example/person_marginals.csv'\n",
    "hh_sample_file = 'data/example/household_sample.csv'\n",
    "person_sample_file = 'data/example/person_sample.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "hh_marg, p_marg, hh_sample, p_sample, xwalk = zs.load_data(hh_marginal_file, person_marginal_file, hh_sample_file, person_sample_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((21, 11), (21, 10), (3149, 10), (7924, 8))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hh_marg.shape, p_marg.shape, hh_sample.shape, p_sample.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>serialno</th>\n",
       "      <th>sample_geog</th>\n",
       "      <th>cars</th>\n",
       "      <th>workers</th>\n",
       "      <th>children</th>\n",
       "      <th>income</th>\n",
       "      <th>VEH</th>\n",
       "      <th>FINCP</th>\n",
       "      <th>NOC</th>\n",
       "      <th>WIF</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2012000013083</td>\n",
       "      <td>1</td>\n",
       "      <td>two or more</td>\n",
       "      <td>none</td>\n",
       "      <td>no</td>\n",
       "      <td>lt35</td>\n",
       "      <td>3</td>\n",
       "      <td>11400.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2012000023543</td>\n",
       "      <td>1</td>\n",
       "      <td>two or more</td>\n",
       "      <td>none</td>\n",
       "      <td>no</td>\n",
       "      <td>lt35</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2012000058856</td>\n",
       "      <td>1</td>\n",
       "      <td>two or more</td>\n",
       "      <td>none</td>\n",
       "      <td>no</td>\n",
       "      <td>lt35</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2012000062123</td>\n",
       "      <td>1</td>\n",
       "      <td>two or more</td>\n",
       "      <td>none</td>\n",
       "      <td>no</td>\n",
       "      <td>lt35</td>\n",
       "      <td>3</td>\n",
       "      <td>30300.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2012000066729</td>\n",
       "      <td>1</td>\n",
       "      <td>two or more</td>\n",
       "      <td>none</td>\n",
       "      <td>no</td>\n",
       "      <td>lt35</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        serialno  sample_geog         cars workers children income  VEH  \\\n",
       "0  2012000013083            1  two or more    none       no   lt35    3   \n",
       "1  2012000023543            1  two or more    none       no   lt35    2   \n",
       "2  2012000058856            1  two or more    none       no   lt35    2   \n",
       "3  2012000062123            1  two or more    none       no   lt35    3   \n",
       "4  2012000066729            1  two or more    none       no   lt35    2   \n",
       "\n",
       "     FINCP  NOC  WIF  \n",
       "0  11400.0    0  0.0  \n",
       "1      NaN    0  NaN  \n",
       "2      NaN    0  NaN  \n",
       "3  30300.0    0  0.0  \n",
       "4      NaN    0  NaN  "
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check sample data\n",
    "hh_sample.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>serialno</th>\n",
       "      <th>sample_geog</th>\n",
       "      <th>age</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>AGEP</th>\n",
       "      <th>SEX</th>\n",
       "      <th>RAC1P</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2012000005576</td>\n",
       "      <td>1</td>\n",
       "      <td>20 to 35</td>\n",
       "      <td>white</td>\n",
       "      <td>male</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2012000017760</td>\n",
       "      <td>1</td>\n",
       "      <td>20 to 35</td>\n",
       "      <td>white</td>\n",
       "      <td>male</td>\n",
       "      <td>34</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2012000021787</td>\n",
       "      <td>1</td>\n",
       "      <td>20 to 35</td>\n",
       "      <td>white</td>\n",
       "      <td>male</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2012000021815</td>\n",
       "      <td>1</td>\n",
       "      <td>20 to 35</td>\n",
       "      <td>white</td>\n",
       "      <td>male</td>\n",
       "      <td>23</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2012000065237</td>\n",
       "      <td>1</td>\n",
       "      <td>20 to 35</td>\n",
       "      <td>white</td>\n",
       "      <td>male</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        serialno  sample_geog       age   race   sex  AGEP  SEX  RAC1P\n",
       "0  2012000005576            1  20 to 35  white  male    27    1      1\n",
       "1  2012000017760            1  20 to 35  white  male    34    1      1\n",
       "2  2012000021787            1  20 to 35  white  male    32    1      1\n",
       "3  2012000021815            1  20 to 35  white  male    23    1      1\n",
       "4  2012000065237            1  20 to 35  white  male    24    1      1"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_sample.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>zone_id</th>\n",
       "      <th colspan=\"3\" halign=\"left\">cars</th>\n",
       "      <th colspan=\"2\" halign=\"left\">children</th>\n",
       "      <th colspan=\"3\" halign=\"left\">income</th>\n",
       "      <th colspan=\"3\" halign=\"left\">workers</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>none</th>\n",
       "      <th>one</th>\n",
       "      <th>two or more</th>\n",
       "      <th>no</th>\n",
       "      <th>yes</th>\n",
       "      <th>gt100</th>\n",
       "      <th>gt35-lt100</th>\n",
       "      <th>lt35</th>\n",
       "      <th>none</th>\n",
       "      <th>one</th>\n",
       "      <th>two or more</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7</td>\n",
       "      <td>49</td>\n",
       "      <td>197</td>\n",
       "      <td>41</td>\n",
       "      <td>215</td>\n",
       "      <td>57</td>\n",
       "      <td>125</td>\n",
       "      <td>74</td>\n",
       "      <td>72</td>\n",
       "      <td>77</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>59</td>\n",
       "      <td>237</td>\n",
       "      <td>68</td>\n",
       "      <td>239</td>\n",
       "      <td>83</td>\n",
       "      <td>126</td>\n",
       "      <td>98</td>\n",
       "      <td>87</td>\n",
       "      <td>93</td>\n",
       "      <td>125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>69</td>\n",
       "      <td>275</td>\n",
       "      <td>79</td>\n",
       "      <td>279</td>\n",
       "      <td>74</td>\n",
       "      <td>170</td>\n",
       "      <td>114</td>\n",
       "      <td>102</td>\n",
       "      <td>108</td>\n",
       "      <td>146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>76</td>\n",
       "      <td>302</td>\n",
       "      <td>167</td>\n",
       "      <td>224</td>\n",
       "      <td>42</td>\n",
       "      <td>105</td>\n",
       "      <td>244</td>\n",
       "      <td>111</td>\n",
       "      <td>118</td>\n",
       "      <td>160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>18</td>\n",
       "      <td>117</td>\n",
       "      <td>466</td>\n",
       "      <td>86</td>\n",
       "      <td>517</td>\n",
       "      <td>50</td>\n",
       "      <td>261</td>\n",
       "      <td>292</td>\n",
       "      <td>171</td>\n",
       "      <td>182</td>\n",
       "      <td>247</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "zone_id cars                  children      income                 workers  \\\n",
       "        none  one two or more       no  yes  gt100 gt35-lt100 lt35    none   \n",
       "1          7   49         197       41  215     57        125   74      72   \n",
       "2          9   59         237       68  239     83        126   98      87   \n",
       "3         10   69         275       79  279     74        170  114     102   \n",
       "4         11   76         302      167  224     42        105  244     111   \n",
       "5         18  117         466       86  517     50        261  292     171   \n",
       "\n",
       "zone_id                   \n",
       "         one two or more  \n",
       "1         77         105  \n",
       "2         93         125  \n",
       "3        108         146  \n",
       "4        118         160  \n",
       "5        182         247  "
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hh_marg.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>zone_id</th>\n",
       "      <th colspan=\"4\" halign=\"left\">age</th>\n",
       "      <th colspan=\"4\" halign=\"left\">race</th>\n",
       "      <th colspan=\"2\" halign=\"left\">sex</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>19 and under</th>\n",
       "      <th>20 to 35</th>\n",
       "      <th>35 to 60</th>\n",
       "      <th>above 60</th>\n",
       "      <th>asian</th>\n",
       "      <th>black</th>\n",
       "      <th>other</th>\n",
       "      <th>white</th>\n",
       "      <th>female</th>\n",
       "      <th>male</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>312</td>\n",
       "      <td>108</td>\n",
       "      <td>223</td>\n",
       "      <td>177</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>756</td>\n",
       "      <td>440</td>\n",
       "      <td>380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>235</td>\n",
       "      <td>143</td>\n",
       "      <td>296</td>\n",
       "      <td>181</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>855</td>\n",
       "      <td>452</td>\n",
       "      <td>403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>303</td>\n",
       "      <td>229</td>\n",
       "      <td>445</td>\n",
       "      <td>174</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>1127</td>\n",
       "      <td>565</td>\n",
       "      <td>586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>215</td>\n",
       "      <td>77</td>\n",
       "      <td>356</td>\n",
       "      <td>189</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>808</td>\n",
       "      <td>389</td>\n",
       "      <td>448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>506</td>\n",
       "      <td>539</td>\n",
       "      <td>619</td>\n",
       "      <td>262</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1926</td>\n",
       "      <td>981</td>\n",
       "      <td>945</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "zone_id          age                             race                    \\\n",
       "        19 and under 20 to 35 35 to 60 above 60 asian black other white   \n",
       "1                312      108      223      177    64     0     0   756   \n",
       "2                235      143      296      181     0     0     0   855   \n",
       "3                303      229      445      174     0     0    24  1127   \n",
       "4                215       77      356      189     0     0    29   808   \n",
       "5                506      539      619      262     0     0     0  1926   \n",
       "\n",
       "zone_id    sex       \n",
       "        female male  \n",
       "1          440  380  \n",
       "2          452  403  \n",
       "3          565  586  \n",
       "4          389  448  \n",
       "5          981  945  "
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_marg.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Synthesize all zones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/danph/Repos/win_ssd/tutorials/urban_ds/synthpop/synthpop/ipu/ipu.py:193: RuntimeWarning: divide by zero encountered in double_scalars\n",
      "  adj = constraint / float((column * weights).sum())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drawing 254 households\n",
      "Drawing 306 households\n",
      "Drawing 356 households\n",
      "Drawing 390 households\n",
      "Drawing 601 households\n",
      "Drawing 324 households\n",
      "Drawing 556 households\n",
      "Drawing 342 households\n",
      "Drawing 273 households\n",
      "Drawing 228 households\n",
      "Drawing 857 households\n",
      "Drawing 748 households\n",
      "Drawing 744 households\n",
      "Drawing 953 households\n",
      "Drawing 719 households\n",
      "Drawing 185 households\n",
      "Drawing 183 households\n",
      "Drawing 286 households\n",
      "Drawing 317 households\n",
      "Drawing 711 households\n",
      "Drawing 345 households\n"
     ]
    }
   ],
   "source": [
    "hh_all, p_all, stats = zs.synthesize_all_zones(hh_marg, p_marg, hh_sample, p_sample, xwalk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((9678, 12), (33921, 11))"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hh_all.shape, p_all.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>serialno</th>\n",
       "      <th>sample_geog</th>\n",
       "      <th>cars</th>\n",
       "      <th>workers</th>\n",
       "      <th>children</th>\n",
       "      <th>income</th>\n",
       "      <th>VEH</th>\n",
       "      <th>FINCP</th>\n",
       "      <th>NOC</th>\n",
       "      <th>WIF</th>\n",
       "      <th>cat_id</th>\n",
       "      <th>geog</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>household_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2013000688302</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>no</td>\n",
       "      <td>lt35</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2010000141368</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>lt35</td>\n",
       "      <td>0</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2012001476110</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>lt35</td>\n",
       "      <td>0</td>\n",
       "      <td>3100.0</td>\n",
       "      <td>3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2010000262947</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>lt35</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2010000141368</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>yes</td>\n",
       "      <td>lt35</td>\n",
       "      <td>0</td>\n",
       "      <td>3500.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   serialno  sample_geog  cars workers children income  VEH  \\\n",
       "household_id                                                                  \n",
       "1             2013000688302            1  none    none       no   lt35    0   \n",
       "2             2010000141368            1  none    none      yes   lt35    0   \n",
       "3             2012001476110            1  none    none      yes   lt35    0   \n",
       "4             2010000262947            1  none    none      yes   lt35    0   \n",
       "5             2010000141368            1  none    none      yes   lt35    0   \n",
       "\n",
       "               FINCP  NOC  WIF  cat_id  geog  \n",
       "household_id                                  \n",
       "1                NaN    0  NaN       6     1  \n",
       "2             3500.0    1  0.0      15     1  \n",
       "3             3100.0    3  0.0      15     1  \n",
       "4                0.0    1  0.0      15     1  \n",
       "5             3500.0    1  0.0      15     1  "
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hh_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>serialno</th>\n",
       "      <th>sample_geog</th>\n",
       "      <th>age</th>\n",
       "      <th>race</th>\n",
       "      <th>sex</th>\n",
       "      <th>AGEP</th>\n",
       "      <th>SEX</th>\n",
       "      <th>RAC1P</th>\n",
       "      <th>cat_id</th>\n",
       "      <th>geog</th>\n",
       "      <th>household_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2012000522864</td>\n",
       "      <td>1</td>\n",
       "      <td>20 to 35</td>\n",
       "      <td>white</td>\n",
       "      <td>male</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>1</td>\n",
       "      <td>189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2012000522864</td>\n",
       "      <td>1</td>\n",
       "      <td>20 to 35</td>\n",
       "      <td>white</td>\n",
       "      <td>female</td>\n",
       "      <td>26</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>68</td>\n",
       "      <td>1</td>\n",
       "      <td>189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2012000522864</td>\n",
       "      <td>1</td>\n",
       "      <td>19 and under</td>\n",
       "      <td>white</td>\n",
       "      <td>female</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2012000522864</td>\n",
       "      <td>1</td>\n",
       "      <td>19 and under</td>\n",
       "      <td>white</td>\n",
       "      <td>female</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>1</td>\n",
       "      <td>189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2012000737274</td>\n",
       "      <td>1</td>\n",
       "      <td>20 to 35</td>\n",
       "      <td>white</td>\n",
       "      <td>male</td>\n",
       "      <td>27</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>69</td>\n",
       "      <td>1</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        serialno  sample_geog           age   race     sex  AGEP  SEX  RAC1P  \\\n",
       "0  2012000522864            1      20 to 35  white    male    27    1      1   \n",
       "1  2012000522864            1      20 to 35  white  female    26    2      1   \n",
       "2  2012000522864            1  19 and under  white  female     5    2      1   \n",
       "3  2012000522864            1  19 and under  white  female     2    2      1   \n",
       "4  2012000737274            1      20 to 35  white    male    27    1      1   \n",
       "\n",
       "   cat_id  geog  household_id  \n",
       "0      69     1           189  \n",
       "1      68     1           189  \n",
       "2      60     1           189  \n",
       "3      60     1           189  \n",
       "4      69     1            86  "
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p_all.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}