stage-3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import sklearn as sk\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "game_csv = pd.read_csv('Training_Dataset.csv',na_values=\"NA\")\n",
    "gameDf = pd.DataFrame(game_csv)\n",
    "\n",
    "dataDict = pd.read_csv('Data_Dictionary.csv',na_values=\"NA\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40000, 52)"
      ]
     },
     "execution_count": 352,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gameDf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1605"
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(gameDf[dic['Income. 0 = No data available']]==0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dic = dict(zip(dataDict.values[:,1], dataDict.values[:,0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gameDf shape: (40000, 52)\n",
      "dataDict shape (52, 2)\n"
     ]
    }
   ],
   "source": [
    "print('gameDf shape:',gameDf.shape)\n",
    "print('dataDict shape', dataDict.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [],
   "source": [
    "del gameDf[dic['Unique customer identifier']]\n",
    "del gameDf['mvar1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50"
      ]
     },
     "execution_count": 357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gameDf.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gameDfBase=pd.DataFrame(pd.read_csv('Training_Dataset.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gameDfBaseTemp = gameDfBase[gameDfBase.mvar9!=0]\n",
    "#gameDfBaseTemp = gameDfBaseTemp[gameDfBaseTemp.mvar9!=0]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(38395, 52)"
      ]
     },
     "execution_count": 360,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gameDfBaseTemp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mvar12</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Apparel</th>\n",
       "      <td>369.0</td>\n",
       "      <td>1.227930e+06</td>\n",
       "      <td>1.565293e+06</td>\n",
       "      <td>18731.0</td>\n",
       "      <td>110976.00</td>\n",
       "      <td>285228.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>9376031.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Automotive</th>\n",
       "      <td>504.0</td>\n",
       "      <td>4.453349e+06</td>\n",
       "      <td>7.944844e+06</td>\n",
       "      <td>38383.0</td>\n",
       "      <td>328233.00</td>\n",
       "      <td>2468209.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Construction</th>\n",
       "      <td>3163.0</td>\n",
       "      <td>1.919864e+06</td>\n",
       "      <td>3.584096e+06</td>\n",
       "      <td>2093.0</td>\n",
       "      <td>157787.00</td>\n",
       "      <td>689301.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Consumables</th>\n",
       "      <td>1122.0</td>\n",
       "      <td>2.021097e+06</td>\n",
       "      <td>4.641765e+06</td>\n",
       "      <td>15606.0</td>\n",
       "      <td>179426.00</td>\n",
       "      <td>490359.5</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Durable goods</th>\n",
       "      <td>1647.0</td>\n",
       "      <td>3.896549e+06</td>\n",
       "      <td>6.844847e+06</td>\n",
       "      <td>11884.0</td>\n",
       "      <td>390937.50</td>\n",
       "      <td>1901520.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Entertainment</th>\n",
       "      <td>2338.0</td>\n",
       "      <td>1.167293e+06</td>\n",
       "      <td>2.046782e+06</td>\n",
       "      <td>1584.0</td>\n",
       "      <td>81415.75</td>\n",
       "      <td>255915.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Finance</th>\n",
       "      <td>2719.0</td>\n",
       "      <td>1.860044e+06</td>\n",
       "      <td>4.016862e+06</td>\n",
       "      <td>9914.0</td>\n",
       "      <td>143516.50</td>\n",
       "      <td>435765.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Household</th>\n",
       "      <td>2308.0</td>\n",
       "      <td>1.669792e+06</td>\n",
       "      <td>3.377517e+06</td>\n",
       "      <td>2863.0</td>\n",
       "      <td>136811.25</td>\n",
       "      <td>499149.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Manufacturing</th>\n",
       "      <td>1772.0</td>\n",
       "      <td>4.001208e+06</td>\n",
       "      <td>6.806952e+06</td>\n",
       "      <td>2730.0</td>\n",
       "      <td>364068.75</td>\n",
       "      <td>2376900.0</td>\n",
       "      <td>3098900.5</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mining</th>\n",
       "      <td>50.0</td>\n",
       "      <td>3.642014e+06</td>\n",
       "      <td>7.360766e+06</td>\n",
       "      <td>41199.0</td>\n",
       "      <td>286032.25</td>\n",
       "      <td>1086603.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Misc services</th>\n",
       "      <td>5346.0</td>\n",
       "      <td>1.403044e+06</td>\n",
       "      <td>2.930383e+06</td>\n",
       "      <td>2017.0</td>\n",
       "      <td>85305.50</td>\n",
       "      <td>208925.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Non durable goods</th>\n",
       "      <td>984.0</td>\n",
       "      <td>4.123568e+06</td>\n",
       "      <td>7.554953e+06</td>\n",
       "      <td>19807.0</td>\n",
       "      <td>308997.00</td>\n",
       "      <td>1841364.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Office supplies</th>\n",
       "      <td>1475.0</td>\n",
       "      <td>1.735099e+06</td>\n",
       "      <td>2.442702e+06</td>\n",
       "      <td>11787.0</td>\n",
       "      <td>100919.00</td>\n",
       "      <td>1267680.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Personal services</th>\n",
       "      <td>2837.0</td>\n",
       "      <td>1.360553e+06</td>\n",
       "      <td>2.877573e+06</td>\n",
       "      <td>792.0</td>\n",
       "      <td>152965.00</td>\n",
       "      <td>349397.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Produce</th>\n",
       "      <td>664.0</td>\n",
       "      <td>1.261397e+06</td>\n",
       "      <td>1.819029e+06</td>\n",
       "      <td>11884.0</td>\n",
       "      <td>90671.00</td>\n",
       "      <td>360569.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>17391440.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Social services</th>\n",
       "      <td>4152.0</td>\n",
       "      <td>1.558220e+06</td>\n",
       "      <td>3.311140e+06</td>\n",
       "      <td>60.0</td>\n",
       "      <td>110922.00</td>\n",
       "      <td>348612.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transport</th>\n",
       "      <td>1511.0</td>\n",
       "      <td>2.125903e+06</td>\n",
       "      <td>4.239841e+06</td>\n",
       "      <td>11806.0</td>\n",
       "      <td>143684.00</td>\n",
       "      <td>633840.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unknown</th>\n",
       "      <td>5434.0</td>\n",
       "      <td>2.774361e+06</td>\n",
       "      <td>8.760516e+05</td>\n",
       "      <td>4753.0</td>\n",
       "      <td>2918974.00</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>2918974.0</td>\n",
       "      <td>34207020.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      count          mean           std      min         25%  \\\n",
       "mvar12                                                                         \n",
       "Apparel               369.0  1.227930e+06  1.565293e+06  18731.0   110976.00   \n",
       "Automotive            504.0  4.453349e+06  7.944844e+06  38383.0   328233.00   \n",
       "Construction         3163.0  1.919864e+06  3.584096e+06   2093.0   157787.00   \n",
       "Consumables          1122.0  2.021097e+06  4.641765e+06  15606.0   179426.00   \n",
       "Durable goods        1647.0  3.896549e+06  6.844847e+06  11884.0   390937.50   \n",
       "Entertainment        2338.0  1.167293e+06  2.046782e+06   1584.0    81415.75   \n",
       "Finance              2719.0  1.860044e+06  4.016862e+06   9914.0   143516.50   \n",
       "Household            2308.0  1.669792e+06  3.377517e+06   2863.0   136811.25   \n",
       "Manufacturing        1772.0  4.001208e+06  6.806952e+06   2730.0   364068.75   \n",
       "Mining                 50.0  3.642014e+06  7.360766e+06  41199.0   286032.25   \n",
       "Misc services        5346.0  1.403044e+06  2.930383e+06   2017.0    85305.50   \n",
       "Non durable goods     984.0  4.123568e+06  7.554953e+06  19807.0   308997.00   \n",
       "Office supplies      1475.0  1.735099e+06  2.442702e+06  11787.0   100919.00   \n",
       "Personal services    2837.0  1.360553e+06  2.877573e+06    792.0   152965.00   \n",
       "Produce               664.0  1.261397e+06  1.819029e+06  11884.0    90671.00   \n",
       "Social services      4152.0  1.558220e+06  3.311140e+06     60.0   110922.00   \n",
       "Transport            1511.0  2.125903e+06  4.239841e+06  11806.0   143684.00   \n",
       "Unknown              5434.0  2.774361e+06  8.760516e+05   4753.0  2918974.00   \n",
       "\n",
       "                           50%        75%         max  \n",
       "mvar12                                                 \n",
       "Apparel               285228.0  2918974.0   9376031.0  \n",
       "Automotive           2468209.0  2918974.0  34207020.0  \n",
       "Construction          689301.0  2918974.0  34207020.0  \n",
       "Consumables           490359.5  2918974.0  34207020.0  \n",
       "Durable goods        1901520.0  2918974.0  34207020.0  \n",
       "Entertainment         255915.0  2918974.0  34207020.0  \n",
       "Finance               435765.0  2918974.0  34207020.0  \n",
       "Household             499149.0  2918974.0  34207020.0  \n",
       "Manufacturing        2376900.0  3098900.5  34207020.0  \n",
       "Mining               1086603.0  2918974.0  34207020.0  \n",
       "Misc services         208925.0  2918974.0  34207020.0  \n",
       "Non durable goods    1841364.0  2918974.0  34207020.0  \n",
       "Office supplies      1267680.0  2918974.0  34207020.0  \n",
       "Personal services     349397.0  2918974.0  34207020.0  \n",
       "Produce               360569.0  2918974.0  17391440.0  \n",
       "Social services       348612.0  2918974.0  34207020.0  \n",
       "Transport             633840.0  2918974.0  34207020.0  \n",
       "Unknown              2918974.0  2918974.0  34207020.0  "
      ]
     },
     "execution_count": 361,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gameDfBaseTemp.groupby('mvar12')['mvar9'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2468209.0"
      ]
     },
     "execution_count": 362,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gameDfBaseTemp.groupby('mvar12')['mvar9'].describe().iloc[1,5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Apparel', 'Automotive         ', 'Construction       ',\n",
       "       'Consumables        ', 'Durable goods      ', 'Entertainment      ',\n",
       "       'Finance            ', 'Household          ', 'Manufacturing      ',\n",
       "       'Mining             ', 'Misc services      ', 'Non durable goods  ',\n",
       "       'Office supplies    ', 'Personal services  ', 'Produce            ',\n",
       "       'Social services    ', 'Transport          ', 'Unknown            '], dtype=object)"
      ]
     },
     "execution_count": 363,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indCode=np.sort(gameDfBaseTemp['mvar12'].unique())\n",
    "indCode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Apparel': 285228.0,\n",
       " 'Automotive         ': 2468209.0,\n",
       " 'Construction       ': 689301.0,\n",
       " 'Consumables        ': 490359.5,\n",
       " 'Durable goods      ': 1901520.0,\n",
       " 'Entertainment      ': 255915.0,\n",
       " 'Finance            ': 435765.0,\n",
       " 'Household          ': 499149.0,\n",
       " 'Manufacturing      ': 2376900.0,\n",
       " 'Mining             ': 1086603.0,\n",
       " 'Misc services      ': 208925.0,\n",
       " 'Non durable goods  ': 1841364.0,\n",
       " 'Office supplies    ': 1267680.0,\n",
       " 'Personal services  ': 349397.0,\n",
       " 'Produce            ': 360569.0,\n",
       " 'Social services    ': 348612.0,\n",
       " 'Transport          ': 633840.0,\n",
       " 'Unknown            ': 2918974.0}"
      ]
     },
     "execution_count": 364,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "medianIncomes=dict(zip(indCode,gameDfBaseTemp.groupby('mvar12')['mvar9'].describe().iloc[:,5]))\n",
    "medianIncomes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Office supplies    '"
      ]
     },
     "execution_count": 365,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(gameDfBase.mvar9==0)\n",
    "gameDfBase.mvar12[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "#imputed income acc to ind code\n",
    "for i in range(len(gameDfBase.mvar9)):\n",
    "    if(gameDfBase.mvar9[i]==0):\n",
    "        gameDfBase.mvar9[i]=medianIncomes[gameDfBase.mvar12[i]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1267680"
      ]
     },
     "execution_count": 367,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(np.sum(gameDfBase.mvar9==0))\n",
    "gameDfBase.mvar9[12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gameDfBase.to_csv('impute1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40000, 52)"
      ]
     },
     "execution_count": 369,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gameDfBase.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#outliers\n",
    "gameDfBaseOutlier = gameDfBase\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar28<=90000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar29<=90000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar30<=90000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar31<=90000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar32<=12000000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar33<=20000000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar34<=20000000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar35<=20000000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar36<=20000000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar37<=5000000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar38<=10000000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar39<=20000000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 373,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar16<=150000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar17<=2200000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar18<=500000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar19<=370000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 374,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar20<=675000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar21<=375000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar22<=300000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar23<=300000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 375,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar24<=150000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar25<=200000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar26<=170000]\n",
    "gameDfBaseOutlier=gameDfBaseOutlier[gameDfBaseOutlier.mvar27<=190000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 376,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gameDfBaseOutlier=gameDfBaseOutlier[ np.log(np.power(gameDfBaseOutlier.mvar8,5)+1)>-4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 377,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(39882, 52)"
      ]
     },
     "execution_count": 377,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gameDfBaseOutlier.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gameDfBaseOutlier.to_csv('outlier1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "features=list(dic.keys())\n",
    "features.remove('Unique customer identifier')\n",
    "features.remove('Card product type')\n",
    "features.remove('Industry code in which the customer has spent the most in past. (Including estimated external).')\n",
    "\n",
    "spef=(dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mvar49\n",
      "0    36553\n",
      "1     3447\n",
      "dtype: int64\n",
      "mvar50\n",
      "0    37346\n",
      "1     2654\n",
      "dtype: int64\n",
      "mvar51\n",
      "0    37223\n",
      "1     2777\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#before outier\n",
    "print(gameDfBase.groupby('mvar49').size())\n",
    "print(gameDfBase.groupby('mvar50').size())\n",
    "print(gameDfBase.groupby('mvar51').size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mvar49\n",
      "0    36462\n",
      "1     3420\n",
      "dtype: int64\n",
      "mvar50\n",
      "0    37233\n",
      "1     2649\n",
      "dtype: int64\n",
      "mvar51\n",
      "0    37108\n",
      "1     2774\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#after outlier\n",
    "print(gameDfBaseOutlier.groupby('mvar49').size())\n",
    "print(gameDfBaseOutlier.groupby('mvar50').size())\n",
    "print(gameDfBaseOutlier.groupby('mvar51').size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24354"
      ]
     },
     "execution_count": 380,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(gameDfBase.mvar3==0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.stats import pearsonr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "f8=np.power(gameDfBaseOutlier.mvar8,5)\n",
    "f2=gameDfBaseOutlier.mvar2\n",
    "f6=gameDfBaseOutlier.mvar6\n",
    "f7=gameDfBaseOutlier.mvar7\n",
    "\n",
    "incomey=gameDfBaseOutlier.mvar9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.72374509733201542, 0.0)"
      ]
     },
     "execution_count": 383,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pearsonr(incomey,f8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        False\n",
       "1        False\n",
       "2        False\n",
       "3        False\n",
       "4        False\n",
       "5        False\n",
       "6        False\n",
       "7        False\n",
       "8        False\n",
       "9        False\n",
       "10       False\n",
       "11       False\n",
       "12        True\n",
       "13       False\n",
       "14       False\n",
       "15       False\n",
       "16       False\n",
       "17       False\n",
       "18       False\n",
       "19       False\n",
       "20       False\n",
       "21       False\n",
       "22       False\n",
       "23       False\n",
       "24       False\n",
       "25       False\n",
       "26       False\n",
       "27       False\n",
       "28       False\n",
       "29       False\n",
       "         ...  \n",
       "39970    False\n",
       "39971    False\n",
       "39972    False\n",
       "39973    False\n",
       "39974    False\n",
       "39975    False\n",
       "39976    False\n",
       "39977    False\n",
       "39978    False\n",
       "39979    False\n",
       "39980    False\n",
       "39981    False\n",
       "39982    False\n",
       "39983    False\n",
       "39984    False\n",
       "39985    False\n",
       "39986    False\n",
       "39987    False\n",
       "39988    False\n",
       "39989    False\n",
       "39990    False\n",
       "39991    False\n",
       "39992    False\n",
       "39993    False\n",
       "39994    False\n",
       "39995    False\n",
       "39996    False\n",
       "39997    False\n",
       "39998    False\n",
       "39999    False\n",
       "Name: mvar9, Length: 40000, dtype: bool"
      ]
     },
     "execution_count": 384,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gameDf.mvar9==0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gameDf=pd.DataFrame(pd.read_csv('Training_Dataset.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 386,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(40000, 52)"
      ]
     },
     "execution_count": 386,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gameDfBase.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#trans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 388,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(39882, 52)"
      ]
     },
     "execution_count": 388,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=gameDfBaseOutlier\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['cm_key', 'mvar1', 'mvar2', 'mvar3', 'mvar4', 'mvar5', 'mvar6', 'mvar7',\n",
       "       'mvar8', 'mvar9', 'mvar10', 'mvar11', 'mvar12', 'mvar13', 'mvar14',\n",
       "       'mvar15', 'mvar16', 'mvar17', 'mvar18', 'mvar19', 'mvar20', 'mvar21',\n",
       "       'mvar22', 'mvar23', 'mvar24', 'mvar25', 'mvar26', 'mvar27', 'mvar28',\n",
       "       'mvar29', 'mvar30', 'mvar31', 'mvar32', 'mvar33', 'mvar34', 'mvar35',\n",
       "       'mvar36', 'mvar37', 'mvar38', 'mvar39', 'mvar40', 'mvar41', 'mvar42',\n",
       "       'mvar43', 'mvar44', 'mvar45', 'mvar46', 'mvar47', 'mvar48', 'mvar49',\n",
       "       'mvar50', 'mvar51'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 389,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "52"
      ]
     },
     "execution_count": 390,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.keys().size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in features:\n",
    "    df[dic[i]] = df[dic[i]].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar16=df.mvar16+df.mvar17+df.mvar18+df.mvar19\n",
    "\n",
    "df.mvar20 = df.mvar20+df.mvar21+df.mvar22+df.mvar23\n",
    "\n",
    "df.mvar24 = df.mvar24+df.mvar25+df.mvar26+df.mvar27\n",
    "\n",
    "df.mvar28 = df.mvar28+df.mvar29+df.mvar30+df.mvar31\n",
    "\n",
    "df.mvar32 = df.mvar32+df.mvar33+df.mvar34+df.mvar35\n",
    "\n",
    "df.mvar36 = df.mvar36+df.mvar37+df.mvar38+df.mvar39"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py:179: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(df.mvar40)-1):\n",
    "    if df.mvar40.iloc[i]!=0:\n",
    "        df.mvar40.iloc[i]=df.mvar43.iloc[i]/df.mvar40.iloc[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 414,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.        ,  1.        ,  0.5       ,  0.66666667,  0.33333333,\n",
       "        0.75      ,  0.25      ,  0.2       ,  0.6       ])"
      ]
     },
     "execution_count": 414,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mvar40.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py:179: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(df.mvar41)-1):\n",
    "    if df.mvar41.iloc[i]!=0.0:\n",
    "        df.mvar41.iloc[i]=df.mvar44.iloc[i]/df.mvar41.iloc[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 397,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.        ,  1.        ,  0.33333333,  0.5       ,  0.75      ,\n",
       "        0.66666667])"
      ]
     },
     "execution_count": 397,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mvar41.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 342,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mvar42.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py:179: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(df.mvar42)-1):\n",
    "    if df.mvar42.iloc[i]!=0:\n",
    "        #print(i)\n",
    "        df.mvar42.iloc[i]= (df.mvar45.iloc[i]/df.mvar42.iloc[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0. ,  1. ,  0.5])"
      ]
     },
     "execution_count": 410,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mvar42.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/anaconda3/lib/python3.6/site-packages/pandas/core/indexing.py:179: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(df.mvar40)-1):\n",
    "    if df.mvar40.iloc[i]>1:\n",
    "        df.mvar40.iloc[i] = 1/df.mvar40.iloc[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 412,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 412,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(df['mvar40']>1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 416,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.75"
      ]
     },
     "execution_count": 416,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mvar40[592]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 417,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar4 = np.log(df.mvar4+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar5 = np.log(df.mvar5+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 419,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar6 = np.log(df.mvar6+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 420,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar7 = np.log(df.mvar7+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 421,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar9 = np.log(df.mvar9+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 422,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar16 = np.log(df.mvar16+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 423,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar17 = np.log(df.mvar17+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 424,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar18 = np.log(df.mvar18+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 425,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar19 = np.log(df.mvar19+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 426,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar20 = np.log(df.mvar20+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 427,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar21 = np.log(df.mvar21+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 428,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar22 = np.log(df.mvar22+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 429,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar23 = np.log(df.mvar23+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 430,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar24 = np.log(df.mvar24+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 431,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar25 = np.log(df.mvar25+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 432,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar27 = np.log(df.mvar27+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 433,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar28 = np.log(df.mvar28+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 434,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar29 = np.log(df.mvar29+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 435,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar30 = np.log(df.mvar30+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 436,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar31 = np.log(df.mvar31+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 437,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar36 = np.log(df.mvar36+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar37 = np.log(df.mvar37+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar38 = np.log(df.mvar38+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar39 = np.log(df.mvar39+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 441,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar32 = np.log(df.mvar32+df.mvar32.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar33 = np.log(df.mvar32+df.mvar33.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 443,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar34 = np.log(df.mvar32+df.mvar34.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 444,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.mvar35 = np.log(df.mvar32+df.mvar35.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 445,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 446,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing as prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 447,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "scaler4 = prep.StandardScaler().fit(df.mvar4)\n",
    "scaler5 = prep.StandardScaler().fit(df.mvar5)\n",
    "scaler6 = prep.StandardScaler().fit(df.mvar6)\n",
    "scaler7 = prep.StandardScaler().fit(df.mvar7)\n",
    "scaler9 = prep.StandardScaler().fit(df.mvar9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 448,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "scaler16 = prep.StandardScaler().fit(df.mvar16)\n",
    "scaler17 = prep.StandardScaler().fit(df.mvar17)\n",
    "scaler18 = prep.StandardScaler().fit(df.mvar18)\n",
    "scaler19 = prep.StandardScaler().fit(df.mvar19)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 449,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "scaler20 = prep.StandardScaler().fit(df.mvar20)\n",
    "scaler21 = prep.StandardScaler().fit(df.mvar21)\n",
    "scaler22 = prep.StandardScaler().fit(df.mvar22)\n",
    "scaler23 = prep.StandardScaler().fit(df.mvar23)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "scaler24 = prep.StandardScaler().fit(df.mvar24)\n",
    "scaler25 = prep.StandardScaler().fit(df.mvar25)\n",
    "scaler26 = prep.StandardScaler().fit(df.mvar26)\n",
    "scaler27 = prep.StandardScaler().fit(df.mvar27)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "scaler28 = prep.StandardScaler().fit(df.mvar28)\n",
    "scaler29 = prep.StandardScaler().fit(df.mvar29)\n",
    "scaler30 = prep.StandardScaler().fit(df.mvar30)\n",
    "scaler31 = prep.StandardScaler().fit(df.mvar31)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 452,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "scaler32 = prep.StandardScaler().fit(df.mvar32)\n",
    "scaler33 = prep.StandardScaler().fit(df.mvar33)\n",
    "scaler34 = prep.StandardScaler().fit(df.mvar34)\n",
    "scaler35 = prep.StandardScaler().fit(df.mvar35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 453,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "scaler36 = prep.StandardScaler().fit(df.mvar36)\n",
    "scaler37 = prep.StandardScaler().fit(df.mvar37)\n",
    "scaler38 = prep.StandardScaler().fit(df.mvar38)\n",
    "scaler39 = prep.StandardScaler().fit(df.mvar39)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 454,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "scaler11 = prep.StandardScaler().fit(df.mvar11)\n",
    "scaler13 = prep.StandardScaler().fit(df.mvar13)\n",
    "scaler2 = prep.StandardScaler().fit(df.mvar2)\n",
    "scaler8 = prep.StandardScaler().fit(df.mvar8)\n",
    "scaler14 = prep.StandardScaler().fit(df.mvar14)\n",
    "scaler15 = prep.StandardScaler().fit(df.mvar15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 455,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:586: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "scaler40 = prep.StandardScaler().fit(df.mvar40)\n",
    "scaler41 = prep.StandardScaler().fit(df.mvar41)\n",
    "scaler42 = prep.StandardScaler().fit(df.mvar42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 456,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#trans 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 457,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.2884382040808262"
      ]
     },
     "execution_count": 457,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mvar4.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 458,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.574132169527291"
      ]
     },
     "execution_count": 458,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mvar4.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 459,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar4=scaler4.transform(df.mvar4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 460,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0000125372199566"
      ]
     },
     "execution_count": 460,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.mvar4.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 461,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar5=scaler5.transform(df.mvar5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 462,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar6=scaler6.transform(df.mvar6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 463,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar7=scaler7.transform(df.mvar7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar9=scaler9.transform(df.mvar9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 465,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar16=scaler16.transform(df.mvar16)\n",
    "df.mvar17=scaler17.transform(df.mvar17)\n",
    "df.mvar18=scaler18.transform(df.mvar18)\n",
    "df.mvar19=scaler19.transform(df.mvar19)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 466,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar20=scaler20.transform(df.mvar20)\n",
    "df.mvar21=scaler21.transform(df.mvar21)\n",
    "df.mvar22=scaler22.transform(df.mvar22)\n",
    "df.mvar23=scaler23.transform(df.mvar23)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 467,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar28=scaler28.transform(df.mvar28)\n",
    "df.mvar29=scaler29.transform(df.mvar29)\n",
    "df.mvar30=scaler30.transform(df.mvar30)\n",
    "df.mvar31=scaler31.transform(df.mvar31)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 468,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar24=scaler24.transform(df.mvar24)\n",
    "df.mvar25=scaler25.transform(df.mvar25)\n",
    "df.mvar26=scaler26.transform(df.mvar26)\n",
    "df.mvar27=scaler27.transform(df.mvar27)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 469,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar36=scaler36.transform(df.mvar36)\n",
    "df.mvar37=scaler37.transform(df.mvar37)\n",
    "df.mvar38=scaler38.transform(df.mvar38)\n",
    "df.mvar39=scaler39.transform(df.mvar39)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 470,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar32=scaler32.transform(df.mvar32)\n",
    "df.mvar33=scaler33.transform(df.mvar33)\n",
    "df.mvar34=scaler34.transform(df.mvar34)\n",
    "df.mvar35=scaler35.transform(df.mvar35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 471,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar11=scaler11.transform(df.mvar11)\n",
    "df.mvar13=scaler13.transform(df.mvar13)\n",
    "df.mvar2=scaler2.transform(df.mvar2)\n",
    "df.mvar8=scaler8.transform(df.mvar8)\n",
    "df.mvar14=scaler14.transform(df.mvar14)\n",
    "df.mvar15=scaler15.transform(df.mvar15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 472,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n",
      "/home/ekka/.local/lib/python3.6/site-packages/sklearn/preprocessing/data.py:649: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  warnings.warn(DEPRECATION_MSG_1D, DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "df.mvar40=scaler40.transform(df.mvar40)\n",
    "df.mvar41=scaler41.transform(df.mvar41)\n",
    "df.mvar42=scaler42.transform(df.mvar42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 473,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del df['mvar43']\n",
    "del df['mvar44']\n",
    "del df['mvar45']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 474,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del df['mvar17']\n",
    "del df['mvar18']\n",
    "del df['mvar19']\n",
    "\n",
    "del df['mvar21']\n",
    "del df['mvar22']\n",
    "del df['mvar23']\n",
    "\n",
    "del df['mvar25']\n",
    "del df['mvar26']\n",
    "del df['mvar27']\n",
    "\n",
    "del df['mvar29']\n",
    "del df['mvar30']\n",
    "del df['mvar31']\n",
    "\n",
    "del df['mvar33']\n",
    "del df['mvar34']\n",
    "del df['mvar35']\n",
    "\n",
    "del df['mvar37']\n",
    "del df['mvar38']\n",
    "del df['mvar39']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 483,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['mvar2', 'mvar3', 'mvar4', 'mvar5', 'mvar6', 'mvar7', 'mvar8', 'mvar9',\n",
       "       'mvar10', 'mvar11', 'mvar12', 'mvar13', 'mvar14', 'mvar15', 'mvar16',\n",
       "       'mvar20', 'mvar24', 'mvar28', 'mvar32', 'mvar36', 'mvar40', 'mvar41',\n",
       "       'mvar42', 'mvar49', 'mvar50', 'mvar51'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 483,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 478,
   "metadata": {},
   "outputs": [],
   "source": [
    "del df['cm_key']\n",
    "del df['mvar1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 484,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del df['mvar12']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 485,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25"
      ]
     },
     "execution_count": 485,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.keys().size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 481,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del df['mvar46']\n",
    "del df['mvar47']\n",
    "del df['mvar48']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 486,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv('clean1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 497,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24290"
      ]
     },
     "execution_count": 497,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(df.mvar3==0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 488,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df2=pd.DataFrame(pd.read_csv('clean1.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 489,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df2=df2[df2.mvar3!=0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 496,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 496,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(df2.mvar3==0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 491,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df2.to_csv('clean2.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 492,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 492,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(df.mvar9==0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 501,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['mvar2', 'mvar3', 'mvar4', 'mvar5', 'mvar6', 'mvar7', 'mvar8', 'mvar9',\n",
       "       'mvar10', 'mvar11', 'mvar13', 'mvar14', 'mvar15', 'mvar16', 'mvar20',\n",
       "       'mvar24', 'mvar28', 'mvar32', 'mvar36', 'mvar40', 'mvar41', 'mvar42',\n",
       "       'mvar49', 'mvar50', 'mvar51'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 501,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df mvar3 =0\n",
    "df.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}