predict_future_sales.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "DLL load failed: The specified module could not be found.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-8273fbea0175>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmetrics\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mr2_score\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mlightgbm\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mlgb\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[0mcatboost\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mCatBoostRegressor\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      9\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtqdm\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mtqdm_notebook\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mitertools\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mproduct\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\syin\\Anaconda3\\lib\\site-packages\\catboost\\__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mcore\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mPool\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mCatBoost\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mCatBoostClassifier\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mCatBoostRegressor\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mCatboostError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcv\u001b[0m  \u001b[1;31m# noqa\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m     \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0mwidget\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mCatboostIpythonWidget\u001b[0m  \u001b[1;31m# noqa\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mexcept\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[1;32mpass\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\Users\\syin\\Anaconda3\\lib\\site-packages\\catboost\\core.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     30\u001b[0m         \u001b[1;32mfrom\u001b[0m \u001b[0m_catboost\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0m_PoolBase\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_CatBoostBase\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mCatboostError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_set_logger\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_reset_logger\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_configure_malloc\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     31\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mImportError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 32\u001b[1;33m         \u001b[1;32mfrom\u001b[0m \u001b[1;33m.\u001b[0m\u001b[0m_catboost\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0m_PoolBase\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_CatBoostBase\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mCatboostError\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_cv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_set_logger\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_reset_logger\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_configure_malloc\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     33\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     34\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mcontextlib\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mcontextmanager\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mImportError\u001b[0m: DLL load failed: The specified module could not be found."
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import r2_score\n",
    "import lightgbm as lgb\n",
    "from catboost import CatBoostRegressor\n",
    "from tqdm import tqdm_notebook\n",
    "from itertools import product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "input_folder = \"../predict_future_sales\" # or input_folder = os.getcwd()\n",
    "file_names = (os.listdir(input_folder))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['.ipynb_checkpoints',\n",
       " 'benchmark.ipynb',\n",
       " 'city_name_eng.csv',\n",
       " 'city_name_rus.csv',\n",
       " 'coordinates.csv',\n",
       " 'EDA.ipynb',\n",
       " 'hol.feather',\n",
       " 'itc_en.feather',\n",
       " 'items.csv',\n",
       " 'item_categories.csv',\n",
       " 'it_en.feather',\n",
       " 'learn_error.tsv',\n",
       " 'meta.tsv',\n",
       " 'predict_future_sales.ipynb',\n",
       " 'Programming_assignment_week_4 (1).ipynb',\n",
       " 'references.txt',\n",
       " 'sales_train.csv.gz',\n",
       " 'sales_train_v2.csv',\n",
       " 'sample_submission.csv',\n",
       " 'sample_submission.csv.gz',\n",
       " 'sample_submission1.csv',\n",
       " 'shops.csv',\n",
       " 'sh_en.feather',\n",
       " 'test.csv',\n",
       " 'test.csv.gz',\n",
       " 'time_left.tsv',\n",
       " 'train']"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sales_train = pd.read_csv('sales_train.csv.gz')\n",
    "shops = pd.read_csv('shops.csv')\n",
    "items = pd.read_csv('items.csv')\n",
    "item_cats = pd.read_csv('item_categories.csv')\n",
    "sample_sub = pd.read_csv('sample_submission.csv.gz')\n",
    "test = pd.read_csv('test.csv.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "hol = pd.read_feather('hol.feather')\n",
    "it_en = pd.read_feather('it_en.feather')\n",
    "itc_en = pd.read_feather('itc_en.feather')\n",
    "sh_en = pd.read_feather('sh_en.feather')\n",
    "coordinates = pd.read_csv('coordinates.csv')\n",
    "city_name_eng = pd.read_csv('city_name_eng.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_id</th>\n",
       "      <th>lat</th>\n",
       "      <th>long</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>62.028103</td>\n",
       "      <td>129.732663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>62.028103</td>\n",
       "      <td>129.732663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>44.429866</td>\n",
       "      <td>40.252969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>55.796339</td>\n",
       "      <td>37.938199</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>48.786293</td>\n",
       "      <td>44.751867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>59.220473</td>\n",
       "      <td>39.891559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>51.661535</td>\n",
       "      <td>39.200287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>51.661535</td>\n",
       "      <td>39.200287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>51.661535</td>\n",
       "      <td>39.200287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>55.597780</td>\n",
       "      <td>38.119497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>55.597780</td>\n",
       "      <td>38.119497</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>55.798551</td>\n",
       "      <td>49.106324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>55.798551</td>\n",
       "      <td>49.106324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>54.513845</td>\n",
       "      <td>36.261215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>55.095240</td>\n",
       "      <td>38.765224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>56.010563</td>\n",
       "      <td>92.852572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>56.010563</td>\n",
       "      <td>92.852572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>51.730361</td>\n",
       "      <td>36.192647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>20</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>21</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>22</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>23</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>24</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>25</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>26</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>27</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>28</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>29</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>30</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>31</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>32</td>\n",
       "      <td>55.753215</td>\n",
       "      <td>37.622504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>33</td>\n",
       "      <td>55.910503</td>\n",
       "      <td>37.736330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>34</td>\n",
       "      <td>56.326887</td>\n",
       "      <td>44.005986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>35</td>\n",
       "      <td>56.326887</td>\n",
       "      <td>44.005986</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>36</td>\n",
       "      <td>55.030199</td>\n",
       "      <td>82.920430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>37</td>\n",
       "      <td>55.030199</td>\n",
       "      <td>82.920430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>38</td>\n",
       "      <td>54.989342</td>\n",
       "      <td>73.368212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>39</td>\n",
       "      <td>47.222543</td>\n",
       "      <td>39.718732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>40</td>\n",
       "      <td>47.222543</td>\n",
       "      <td>39.718732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>41</td>\n",
       "      <td>47.222543</td>\n",
       "      <td>39.718732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>42</td>\n",
       "      <td>59.939095</td>\n",
       "      <td>30.315868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>43</td>\n",
       "      <td>59.939095</td>\n",
       "      <td>30.315868</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>44</td>\n",
       "      <td>53.195538</td>\n",
       "      <td>50.101783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>45</td>\n",
       "      <td>53.195538</td>\n",
       "      <td>50.101783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>46</td>\n",
       "      <td>56.315321</td>\n",
       "      <td>38.135766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>47</td>\n",
       "      <td>61.254035</td>\n",
       "      <td>73.396221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>48</td>\n",
       "      <td>56.484660</td>\n",
       "      <td>84.948179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>49</td>\n",
       "      <td>57.153033</td>\n",
       "      <td>65.534328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>50</td>\n",
       "      <td>57.153033</td>\n",
       "      <td>65.534328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>51</td>\n",
       "      <td>57.153033</td>\n",
       "      <td>65.534328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>52</td>\n",
       "      <td>54.735147</td>\n",
       "      <td>55.958727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>53</td>\n",
       "      <td>54.735147</td>\n",
       "      <td>55.958727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>54</td>\n",
       "      <td>55.888796</td>\n",
       "      <td>37.430328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>55</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>56</td>\n",
       "      <td>55.149851</td>\n",
       "      <td>37.466997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>57</td>\n",
       "      <td>62.028103</td>\n",
       "      <td>129.732663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>58</td>\n",
       "      <td>62.028103</td>\n",
       "      <td>129.732663</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>59</td>\n",
       "      <td>57.626569</td>\n",
       "      <td>39.893787</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    shop_id        lat        long\n",
       "0         0  62.028103  129.732663\n",
       "1         1  62.028103  129.732663\n",
       "2         2  44.429866   40.252969\n",
       "3         3  55.796339   37.938199\n",
       "4         4  48.786293   44.751867\n",
       "5         5  59.220473   39.891559\n",
       "6         6  51.661535   39.200287\n",
       "7         7  51.661535   39.200287\n",
       "8         8  51.661535   39.200287\n",
       "9         9        NaN         NaN\n",
       "10       10  55.597780   38.119497\n",
       "11       11  55.597780   38.119497\n",
       "12       12        NaN         NaN\n",
       "13       13  55.798551   49.106324\n",
       "14       14  55.798551   49.106324\n",
       "15       15  54.513845   36.261215\n",
       "16       16  55.095240   38.765224\n",
       "17       17  56.010563   92.852572\n",
       "18       18  56.010563   92.852572\n",
       "19       19  51.730361   36.192647\n",
       "20       20  55.753215   37.622504\n",
       "21       21  55.753215   37.622504\n",
       "22       22  55.753215   37.622504\n",
       "23       23  55.753215   37.622504\n",
       "24       24  55.753215   37.622504\n",
       "25       25  55.753215   37.622504\n",
       "26       26  55.753215   37.622504\n",
       "27       27  55.753215   37.622504\n",
       "28       28  55.753215   37.622504\n",
       "29       29  55.753215   37.622504\n",
       "30       30  55.753215   37.622504\n",
       "31       31  55.753215   37.622504\n",
       "32       32  55.753215   37.622504\n",
       "33       33  55.910503   37.736330\n",
       "34       34  56.326887   44.005986\n",
       "35       35  56.326887   44.005986\n",
       "36       36  55.030199   82.920430\n",
       "37       37  55.030199   82.920430\n",
       "38       38  54.989342   73.368212\n",
       "39       39  47.222543   39.718732\n",
       "40       40  47.222543   39.718732\n",
       "41       41  47.222543   39.718732\n",
       "42       42  59.939095   30.315868\n",
       "43       43  59.939095   30.315868\n",
       "44       44  53.195538   50.101783\n",
       "45       45  53.195538   50.101783\n",
       "46       46  56.315321   38.135766\n",
       "47       47  61.254035   73.396221\n",
       "48       48  56.484660   84.948179\n",
       "49       49  57.153033   65.534328\n",
       "50       50  57.153033   65.534328\n",
       "51       51  57.153033   65.534328\n",
       "52       52  54.735147   55.958727\n",
       "53       53  54.735147   55.958727\n",
       "54       54  55.888796   37.430328\n",
       "55       55        NaN         NaN\n",
       "56       56  55.149851   37.466997\n",
       "57       57  62.028103  129.732663\n",
       "58       58  62.028103  129.732663\n",
       "59       59  57.626569   39.893787"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coordinates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sname</th>\n",
       "      <th>sid</th>\n",
       "      <th>en_sname</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Якутск Орджоникидзе 56 фран</td>\n",
       "      <td>0</td>\n",
       "      <td>Yakutsk Ordzhonikidze 56 fr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Якутск ТЦ Центральный фран</td>\n",
       "      <td>1</td>\n",
       "      <td>Yakutsk Central Business Center</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Адыгея ТЦ Мега</td>\n",
       "      <td>2</td>\n",
       "      <td>Adygea Shopping Center Mega</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Балашиха ТРК Октябрь Киномир</td>\n",
       "      <td>3</td>\n",
       "      <td>Balashikha TRK October Kinomir</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Волжский ТЦ Волга Молл</td>\n",
       "      <td>4</td>\n",
       "      <td>Volzhsky shopping center Volga Mall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Вологда ТРЦ Мармелад</td>\n",
       "      <td>5</td>\n",
       "      <td>Vologda Shopping Center Marmalade</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Воронеж Плехановская 13</td>\n",
       "      <td>6</td>\n",
       "      <td>Voronezh Plekhanovskaya 13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Воронеж ТРЦ Максимир</td>\n",
       "      <td>7</td>\n",
       "      <td>Voronezh SEC Maksimir</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Воронеж ТРЦ Сити Парк Град</td>\n",
       "      <td>8</td>\n",
       "      <td>Voronezh SEC City Park Grad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Выездная Торговля</td>\n",
       "      <td>9</td>\n",
       "      <td>Outbound Trade</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Жуковский ул Чкалова 39м</td>\n",
       "      <td>10</td>\n",
       "      <td>Zhukovsky st Chkalov 39m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Жуковский ул Чкалова 39м²</td>\n",
       "      <td>11</td>\n",
       "      <td>Zhukovsky str. Chkalov 39 m²</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Интернет магазин ЧС</td>\n",
       "      <td>12</td>\n",
       "      <td>Online shop Emergency</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Казань ТЦ Бехетле</td>\n",
       "      <td>13</td>\n",
       "      <td>Kazan TC Behetlet</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Казань ТЦ ПаркХаус II</td>\n",
       "      <td>14</td>\n",
       "      <td>Kazan TC ParkHouse II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Калуга ТРЦ XXI век</td>\n",
       "      <td>15</td>\n",
       "      <td>Kaluga SEC XXI century</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Коломна ТЦ Рио</td>\n",
       "      <td>16</td>\n",
       "      <td>Kolomna Shopping Center Rio</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Красноярск ТЦ Взлетка Плаза</td>\n",
       "      <td>17</td>\n",
       "      <td>Krasnoyarsk Shopping Center Vzletka Plaza</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Красноярск ТЦ Июнь</td>\n",
       "      <td>18</td>\n",
       "      <td>Krasnoyarsk Shopping Center June</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Курск ТЦ Пушкинский</td>\n",
       "      <td>19</td>\n",
       "      <td>Kursk Shopping Center Pushkinsky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Москва Распродажа</td>\n",
       "      <td>20</td>\n",
       "      <td>Moscow Sale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Москва МТРЦ Афи Молл</td>\n",
       "      <td>21</td>\n",
       "      <td>Moscow MTRC Afi Mall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Москва Магазин С21</td>\n",
       "      <td>22</td>\n",
       "      <td>Moscow Shop С21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Москва ТК Буденовский пав А2</td>\n",
       "      <td>23</td>\n",
       "      <td>Moscow TC Budyonovsky PAV A2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Москва ТК Буденовский пав К7</td>\n",
       "      <td>24</td>\n",
       "      <td>Moscow TC Budenovskiy Pavil K7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Москва ТРК Атриум</td>\n",
       "      <td>25</td>\n",
       "      <td>Moscow TRK Atrium</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Москва ТЦ Ареал Беляево</td>\n",
       "      <td>26</td>\n",
       "      <td>Moscow TC Areal Belyaevo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Москва ТЦ МЕГА Белая Дача II</td>\n",
       "      <td>27</td>\n",
       "      <td>Moscow Shopping Center MEGA Belaya Dacha II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Москва ТЦ МЕГА Теплый Стан II</td>\n",
       "      <td>28</td>\n",
       "      <td>Moscow TC MEGA Teply Stan II</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Москва ТЦ Новый век Новокосино</td>\n",
       "      <td>29</td>\n",
       "      <td>Moscow TC The New Century of Novokosino</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Москва ТЦ Перловский</td>\n",
       "      <td>30</td>\n",
       "      <td>Moscow Perlovsky shopping center</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Москва ТЦ Семеновский</td>\n",
       "      <td>31</td>\n",
       "      <td>Moscow TC Semenovskiy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Москва ТЦ Серебряный Дом</td>\n",
       "      <td>32</td>\n",
       "      <td>Moscow Shopping Center Silver House</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Мытищи ТРК XL 3</td>\n",
       "      <td>33</td>\n",
       "      <td>Mytisci TRK XL 3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Н Новгород ТРЦ РИО</td>\n",
       "      <td>34</td>\n",
       "      <td>Н Novgorod TRC RIO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Н Новгород ТРЦ Фантастика</td>\n",
       "      <td>35</td>\n",
       "      <td>Н Новгород Shopping and entertainment center F...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Новосибирск ТРЦ Галерея Новосибирск</td>\n",
       "      <td>36</td>\n",
       "      <td>Novosibirsk TRC Gallery Novosibirsk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Новосибирск ТЦ Мега</td>\n",
       "      <td>37</td>\n",
       "      <td>Novosibirsk Shopping Center Mega</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Омск ТЦ Мега</td>\n",
       "      <td>38</td>\n",
       "      <td>Omsk Shopping Center Mega</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>РостовНаДону ТРК Мегацентр Горизонт</td>\n",
       "      <td>39</td>\n",
       "      <td>RostovNaDonu TRK Megacenter Horizon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>РостовНаДону ТРК Мегацентр Горизонт Островной</td>\n",
       "      <td>40</td>\n",
       "      <td>RostovNaDonu TRK Megacenter Horizon Ostrovnoy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>РостовНаДону ТЦ Мега</td>\n",
       "      <td>41</td>\n",
       "      <td>RostovNaDonu Shopping Center Mega</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>СПб ТК Невский Центр</td>\n",
       "      <td>42</td>\n",
       "      <td>SPb TC Nevsky Center</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>СПб ТК Сенная</td>\n",
       "      <td>43</td>\n",
       "      <td>SPb TK Sennaya</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Самара ТЦ Мелодия</td>\n",
       "      <td>44</td>\n",
       "      <td>Samara TC Melody</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Самара ТЦ ПаркХаус</td>\n",
       "      <td>45</td>\n",
       "      <td>Samara TC ParkHouse</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Сергиев Посад ТЦ 7Я</td>\n",
       "      <td>46</td>\n",
       "      <td>Sergiev Posad ТЦ 7Я</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Сургут ТРЦ Сити Молл</td>\n",
       "      <td>47</td>\n",
       "      <td>Surgut Shopping Mall City Mall</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Томск ТРЦ Изумрудный Город</td>\n",
       "      <td>48</td>\n",
       "      <td>Tomsk Shopping Mall Emerald City</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Тюмень ТРЦ Кристалл</td>\n",
       "      <td>49</td>\n",
       "      <td>Tyumen SEC Crystal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Тюмень ТЦ Гудвин</td>\n",
       "      <td>50</td>\n",
       "      <td>Tyumen TC Goodwin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Тюмень ТЦ Зеленый Берег</td>\n",
       "      <td>51</td>\n",
       "      <td>Tyumen Shopping Center Green Beach</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>Уфа ТК Центральный</td>\n",
       "      <td>52</td>\n",
       "      <td>Ufa TC Central</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>Уфа ТЦ Семья 2</td>\n",
       "      <td>53</td>\n",
       "      <td>Ufa TC Family 2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Химки ТЦ Мега</td>\n",
       "      <td>54</td>\n",
       "      <td>Khimki Shopping Center Mega</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>Цифровой склад 1С Онлайн</td>\n",
       "      <td>55</td>\n",
       "      <td>Digital warehouse 1C Online</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Чехов ТРЦ Карнавал</td>\n",
       "      <td>56</td>\n",
       "      <td>Chekhov SEC Carnival</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>Якутск Орджоникидзе 56</td>\n",
       "      <td>57</td>\n",
       "      <td>Yakutsk Ordzhonikidze 56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>Якутск ТЦ Центральный</td>\n",
       "      <td>58</td>\n",
       "      <td>Yakutsk Trade Center</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>Ярославль ТЦ Альтаир</td>\n",
       "      <td>59</td>\n",
       "      <td>Yaroslavl Shopping Center Altair</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sname  sid  \\\n",
       "0                     Якутск Орджоникидзе 56 фран    0   \n",
       "1                      Якутск ТЦ Центральный фран    1   \n",
       "2                                  Адыгея ТЦ Мега    2   \n",
       "3                    Балашиха ТРК Октябрь Киномир    3   \n",
       "4                          Волжский ТЦ Волга Молл    4   \n",
       "5                            Вологда ТРЦ Мармелад    5   \n",
       "6                         Воронеж Плехановская 13    6   \n",
       "7                            Воронеж ТРЦ Максимир    7   \n",
       "8                      Воронеж ТРЦ Сити Парк Град    8   \n",
       "9                               Выездная Торговля    9   \n",
       "10                       Жуковский ул Чкалова 39м   10   \n",
       "11                      Жуковский ул Чкалова 39м²   11   \n",
       "12                            Интернет магазин ЧС   12   \n",
       "13                              Казань ТЦ Бехетле   13   \n",
       "14                          Казань ТЦ ПаркХаус II   14   \n",
       "15                             Калуга ТРЦ XXI век   15   \n",
       "16                                 Коломна ТЦ Рио   16   \n",
       "17                    Красноярск ТЦ Взлетка Плаза   17   \n",
       "18                             Красноярск ТЦ Июнь   18   \n",
       "19                            Курск ТЦ Пушкинский   19   \n",
       "20                              Москва Распродажа   20   \n",
       "21                           Москва МТРЦ Афи Молл   21   \n",
       "22                             Москва Магазин С21   22   \n",
       "23                   Москва ТК Буденовский пав А2   23   \n",
       "24                   Москва ТК Буденовский пав К7   24   \n",
       "25                              Москва ТРК Атриум   25   \n",
       "26                        Москва ТЦ Ареал Беляево   26   \n",
       "27                   Москва ТЦ МЕГА Белая Дача II   27   \n",
       "28                  Москва ТЦ МЕГА Теплый Стан II   28   \n",
       "29                 Москва ТЦ Новый век Новокосино   29   \n",
       "30                           Москва ТЦ Перловский   30   \n",
       "31                          Москва ТЦ Семеновский   31   \n",
       "32                       Москва ТЦ Серебряный Дом   32   \n",
       "33                                Мытищи ТРК XL 3   33   \n",
       "34                             Н Новгород ТРЦ РИО   34   \n",
       "35                      Н Новгород ТРЦ Фантастика   35   \n",
       "36            Новосибирск ТРЦ Галерея Новосибирск   36   \n",
       "37                            Новосибирск ТЦ Мега   37   \n",
       "38                                   Омск ТЦ Мега   38   \n",
       "39            РостовНаДону ТРК Мегацентр Горизонт   39   \n",
       "40  РостовНаДону ТРК Мегацентр Горизонт Островной   40   \n",
       "41                           РостовНаДону ТЦ Мега   41   \n",
       "42                           СПб ТК Невский Центр   42   \n",
       "43                                  СПб ТК Сенная   43   \n",
       "44                              Самара ТЦ Мелодия   44   \n",
       "45                             Самара ТЦ ПаркХаус   45   \n",
       "46                            Сергиев Посад ТЦ 7Я   46   \n",
       "47                           Сургут ТРЦ Сити Молл   47   \n",
       "48                     Томск ТРЦ Изумрудный Город   48   \n",
       "49                            Тюмень ТРЦ Кристалл   49   \n",
       "50                               Тюмень ТЦ Гудвин   50   \n",
       "51                        Тюмень ТЦ Зеленый Берег   51   \n",
       "52                             Уфа ТК Центральный   52   \n",
       "53                                 Уфа ТЦ Семья 2   53   \n",
       "54                                  Химки ТЦ Мега   54   \n",
       "55                       Цифровой склад 1С Онлайн   55   \n",
       "56                             Чехов ТРЦ Карнавал   56   \n",
       "57                         Якутск Орджоникидзе 56   57   \n",
       "58                          Якутск ТЦ Центральный   58   \n",
       "59                           Ярославль ТЦ Альтаир   59   \n",
       "\n",
       "                                             en_sname  \n",
       "0                         Yakutsk Ordzhonikidze 56 fr  \n",
       "1                     Yakutsk Central Business Center  \n",
       "2                         Adygea Shopping Center Mega  \n",
       "3                      Balashikha TRK October Kinomir  \n",
       "4                 Volzhsky shopping center Volga Mall  \n",
       "5                   Vologda Shopping Center Marmalade  \n",
       "6                          Voronezh Plekhanovskaya 13  \n",
       "7                               Voronezh SEC Maksimir  \n",
       "8                         Voronezh SEC City Park Grad  \n",
       "9                                      Outbound Trade  \n",
       "10                           Zhukovsky st Chkalov 39m  \n",
       "11                       Zhukovsky str. Chkalov 39 m²  \n",
       "12                              Online shop Emergency  \n",
       "13                                  Kazan TC Behetlet  \n",
       "14                              Kazan TC ParkHouse II  \n",
       "15                             Kaluga SEC XXI century  \n",
       "16                        Kolomna Shopping Center Rio  \n",
       "17          Krasnoyarsk Shopping Center Vzletka Plaza  \n",
       "18                   Krasnoyarsk Shopping Center June  \n",
       "19                   Kursk Shopping Center Pushkinsky  \n",
       "20                                        Moscow Sale  \n",
       "21                               Moscow MTRC Afi Mall  \n",
       "22                                    Moscow Shop С21  \n",
       "23                       Moscow TC Budyonovsky PAV A2  \n",
       "24                     Moscow TC Budenovskiy Pavil K7  \n",
       "25                                  Moscow TRK Atrium  \n",
       "26                           Moscow TC Areal Belyaevo  \n",
       "27        Moscow Shopping Center MEGA Belaya Dacha II  \n",
       "28                       Moscow TC MEGA Teply Stan II  \n",
       "29            Moscow TC The New Century of Novokosino  \n",
       "30                   Moscow Perlovsky shopping center  \n",
       "31                              Moscow TC Semenovskiy  \n",
       "32                Moscow Shopping Center Silver House  \n",
       "33                                   Mytisci TRK XL 3  \n",
       "34                                 Н Novgorod TRC RIO  \n",
       "35  Н Новгород Shopping and entertainment center F...  \n",
       "36                Novosibirsk TRC Gallery Novosibirsk  \n",
       "37                   Novosibirsk Shopping Center Mega  \n",
       "38                          Omsk Shopping Center Mega  \n",
       "39                RostovNaDonu TRK Megacenter Horizon  \n",
       "40      RostovNaDonu TRK Megacenter Horizon Ostrovnoy  \n",
       "41                  RostovNaDonu Shopping Center Mega  \n",
       "42                               SPb TC Nevsky Center  \n",
       "43                                     SPb TK Sennaya  \n",
       "44                                   Samara TC Melody  \n",
       "45                                Samara TC ParkHouse  \n",
       "46                                Sergiev Posad ТЦ 7Я  \n",
       "47                     Surgut Shopping Mall City Mall  \n",
       "48                   Tomsk Shopping Mall Emerald City  \n",
       "49                                 Tyumen SEC Crystal  \n",
       "50                                  Tyumen TC Goodwin  \n",
       "51                 Tyumen Shopping Center Green Beach  \n",
       "52                                     Ufa TC Central  \n",
       "53                                    Ufa TC Family 2  \n",
       "54                        Khimki Shopping Center Mega  \n",
       "55                        Digital warehouse 1C Online  \n",
       "56                               Chekhov SEC Carnival  \n",
       "57                           Yakutsk Ordzhonikidze 56  \n",
       "58                               Yakutsk Trade Center  \n",
       "59                   Yaroslavl Shopping Center Altair  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sh_en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_name</th>\n",
       "      <th>shop_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>!Якутск Орджоникидзе, 56 фран</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>!Якутск ТЦ \"Центральный\" фран</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Адыгея ТЦ \"Мега\"</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Балашиха ТРК \"Октябрь-Киномир\"</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Волжский ТЦ \"Волга Молл\"</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        shop_name  shop_id\n",
       "0   !Якутск Орджоникидзе, 56 фран        0\n",
       "1   !Якутск ТЦ \"Центральный\" фран        1\n",
       "2                Адыгея ТЦ \"Мега\"        2\n",
       "3  Балашиха ТРК \"Октябрь-Киномир\"        3\n",
       "4        Волжский ТЦ \"Волга Молл\"        4"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shops.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>item_cnt_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  item_cnt_month\n",
       "0   0             0.5\n",
       "1   1             0.5\n",
       "2   2             0.5\n",
       "3   3             0.5\n",
       "4   4             0.5"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "test['item_cnt_day'] = -1\n",
    "test['date_block_num'] = 34\n",
    "sales_train['ID'] = 999"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_cnt_day</th>\n",
       "      <th>date_block_num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>5037</td>\n",
       "      <td>-1</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>5320</td>\n",
       "      <td>-1</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>5233</td>\n",
       "      <td>-1</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5232</td>\n",
       "      <td>-1</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5268</td>\n",
       "      <td>-1</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  shop_id  item_id  item_cnt_day  date_block_num\n",
       "0   0        5     5037            -1              34\n",
       "1   1        5     5320            -1              34\n",
       "2   2        5     5233            -1              34\n",
       "3   3        5     5232            -1              34\n",
       "4   4        5     5268            -1              34"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>item_price</th>\n",
       "      <th>item_cnt_day</th>\n",
       "      <th>ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>02.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>59</td>\n",
       "      <td>22154</td>\n",
       "      <td>999.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>03.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2552</td>\n",
       "      <td>899.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>05.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2552</td>\n",
       "      <td>899.00</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>06.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2554</td>\n",
       "      <td>1709.05</td>\n",
       "      <td>1.0</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>15.01.2013</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>2555</td>\n",
       "      <td>1099.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>999</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         date  date_block_num  shop_id  item_id  item_price  item_cnt_day   ID\n",
       "0  02.01.2013               0       59    22154      999.00           1.0  999\n",
       "1  03.01.2013               0       25     2552      899.00           1.0  999\n",
       "2  05.01.2013               0       25     2552      899.00          -1.0  999\n",
       "3  06.01.2013               0       25     2554     1709.05           1.0  999\n",
       "4  15.01.2013               0       25     2555     1099.00           1.0  999"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sales_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def downcast_dtypes(df):\n",
    "    '''\n",
    "        Changes column types in the dataframe: \n",
    "                `float64` type to `float32`\n",
    "                `int64`   type to `int32`\n",
    "    '''\n",
    "\n",
    "    # Select columns to downcast\n",
    "    float_cols = [c for c in df if df[c].dtype == \"float64\"]\n",
    "    int_cols =   [c for c in df if df[c].dtype == \"int64\"]\n",
    " \n",
    "    # Downcast\n",
    "    df[float_cols] = df[float_cols].astype(np.float32)\n",
    "    df[int_cols]   = df[int_cols].astype(np.int32)\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sales = sales_train[['ID','date_block_num','shop_id','item_id','item_cnt_day']]\n",
    "test = test[['ID','date_block_num','shop_id','item_id','item_cnt_day']]\n",
    "sales = pd.concat([sales, test])\n",
    "# sales = sales[sales['shop_id'].isin([26, 27, 28])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\syin\\Anaconda3\\lib\\site-packages\\pandas\\core\\groupby.py:4291: FutureWarning: using a dict with renaming is deprecated and will be removed in a future version\n",
      "  return super(DataFrameGroupBy, self).aggregate(arg, *args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "# Create \"grid\" with columns\n",
    "index_cols = ['shop_id', 'item_id', 'date_block_num']\n",
    "\n",
    "# For every month we create a grid from all shops/items combinations from that month\n",
    "grid = [] \n",
    "for block_num in sales['date_block_num'].unique():\n",
    "    cur_shops = sales.loc[sales['date_block_num'] == block_num, 'shop_id'].unique()\n",
    "    cur_items = sales.loc[sales['date_block_num'] == block_num, 'item_id'].unique()\n",
    "    grid.append(np.array(list(product(*[cur_shops, cur_items, [block_num]])),dtype='int32'))\n",
    "\n",
    "# Turn the grid into a dataframe\n",
    "grid = pd.DataFrame(np.vstack(grid), columns = index_cols,dtype=np.int32)\n",
    "\n",
    "# Groupby data to get shop-item-month aggregates\n",
    "gb = sales.groupby(index_cols,as_index=False).agg({'item_cnt_day':{'target':'sum'}})\n",
    "# Fix column names\n",
    "gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values] \n",
    "# Join it to the grid\n",
    "all_data = pd.merge(grid, gb, how='left', on=index_cols).fillna(0)\n",
    "\n",
    "# Same as above but with shop-month aggregates\n",
    "gb = sales.groupby(['shop_id', 'date_block_num'],as_index=False).agg({'item_cnt_day':{'target_shop':'sum'}})\n",
    "gb.columns = [col[0] if col[-1]=='' else col[-1] for col in gb.columns.values]\n",
    "all_data = pd.merge(all_data, gb, how='left', on=['shop_id', 'date_block_num']).fillna(0)\n",
    "\n",
    "# Same as above but with item-month aggregates\n",
    "gb = sales.groupby(['item_id', 'date_block_num'],as_index=False).agg({'item_cnt_day':{'target_item':'sum'}})\n",
    "gb.columns = [col[0] if col[-1] == '' else col[-1] for col in gb.columns.values]\n",
    "all_data = pd.merge(all_data, gb, how='left', on=['item_id', 'date_block_num']).fillna(0)\n",
    "\n",
    "# Downcast dtypes from 64 to 32 bit to save memory\n",
    "all_data = downcast_dtypes(all_data)\n",
    "del grid, gb \n",
    "gc.collect();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>target</th>\n",
       "      <th>target_shop</th>\n",
       "      <th>target_item</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10913845</th>\n",
       "      <td>21</td>\n",
       "      <td>7635</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1912.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10913846</th>\n",
       "      <td>21</td>\n",
       "      <td>7638</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1912.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10913847</th>\n",
       "      <td>21</td>\n",
       "      <td>7640</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1912.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10913848</th>\n",
       "      <td>21</td>\n",
       "      <td>7632</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1912.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10913849</th>\n",
       "      <td>21</td>\n",
       "      <td>7440</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1912.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          shop_id  item_id  date_block_num  target  target_shop  target_item\n",
       "10913845       21     7635              33     0.0       1912.0          1.0\n",
       "10913846       21     7638              33     0.0       1912.0          1.0\n",
       "10913847       21     7640              33     0.0       1912.0          1.0\n",
       "10913848       21     7632              33     0.0       1912.0          1.0\n",
       "10913849       21     7440              33     0.0       1912.0          1.0"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8c819e06c9f43fbab076940d555f50d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=6), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# List of columns that we will use to create lags\n",
    "cols_to_rename = list(all_data.columns.difference(index_cols)) \n",
    "\n",
    "shift_range = [1, 2, 3, 4, 5, 12]\n",
    "\n",
    "for month_shift in tqdm_notebook(shift_range):\n",
    "    train_shift = all_data[index_cols + cols_to_rename].copy()\n",
    "    \n",
    "    train_shift['date_block_num'] = train_shift['date_block_num'] + month_shift\n",
    "    \n",
    "    foo = lambda x: '{}_lag_{}'.format(x, month_shift) if x in cols_to_rename else x\n",
    "    train_shift = train_shift.rename(columns=foo)\n",
    "\n",
    "    all_data = pd.merge(all_data, train_shift, on=index_cols, how='left').fillna(0)\n",
    "\n",
    "del train_shift\n",
    "\n",
    "# Don't use old data from year 2013\n",
    "all_data = all_data[all_data['date_block_num'] >= 12] \n",
    "\n",
    "# List of all lagged features\n",
    "fit_cols = [col for col in all_data.columns if col[-1] in [str(item) for item in shift_range]] \n",
    "# We will drop these at fitting stage\n",
    "to_drop_cols = list(set(list(all_data.columns)) - (set(fit_cols)|set(index_cols))) + ['date_block_num'] \n",
    "\n",
    "# Category for each item\n",
    "item_category_mapping = items[['item_id','item_category_id']].drop_duplicates()\n",
    "\n",
    "all_data = pd.merge(all_data, item_category_mapping, how='left', on='item_id')\n",
    "all_data = downcast_dtypes(all_data)\n",
    "gc.collect();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['target_item', 'target_shop', 'target', 'date_block_num']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_drop_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>date_block_num</th>\n",
       "      <th>target</th>\n",
       "      <th>target_shop</th>\n",
       "      <th>target_item</th>\n",
       "      <th>target_lag_1</th>\n",
       "      <th>target_item_lag_1</th>\n",
       "      <th>target_shop_lag_1</th>\n",
       "      <th>target_lag_2</th>\n",
       "      <th>...</th>\n",
       "      <th>target_lag_4</th>\n",
       "      <th>target_item_lag_4</th>\n",
       "      <th>target_shop_lag_4</th>\n",
       "      <th>target_lag_5</th>\n",
       "      <th>target_item_lag_5</th>\n",
       "      <th>target_shop_lag_5</th>\n",
       "      <th>target_lag_12</th>\n",
       "      <th>target_item_lag_12</th>\n",
       "      <th>target_shop_lag_12</th>\n",
       "      <th>item_category_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6425089</th>\n",
       "      <td>21</td>\n",
       "      <td>7635</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1912.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6425090</th>\n",
       "      <td>21</td>\n",
       "      <td>7638</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1912.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6425091</th>\n",
       "      <td>21</td>\n",
       "      <td>7640</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1912.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6425092</th>\n",
       "      <td>21</td>\n",
       "      <td>7632</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1912.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1900.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6425093</th>\n",
       "      <td>21</td>\n",
       "      <td>7440</td>\n",
       "      <td>33</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1912.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1900.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1844.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1717.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         shop_id  item_id  date_block_num  target  target_shop  target_item  \\\n",
       "6425089       21     7635              33     0.0       1912.0          1.0   \n",
       "6425090       21     7638              33     0.0       1912.0          1.0   \n",
       "6425091       21     7640              33     0.0       1912.0          1.0   \n",
       "6425092       21     7632              33     0.0       1912.0          1.0   \n",
       "6425093       21     7440              33     0.0       1912.0          1.0   \n",
       "\n",
       "         target_lag_1  target_item_lag_1  target_shop_lag_1  target_lag_2  \\\n",
       "6425089           0.0                0.0                0.0           0.0   \n",
       "6425090           0.0                0.0                0.0           0.0   \n",
       "6425091           0.0                0.0                0.0           0.0   \n",
       "6425092           0.0                1.0             1900.0           0.0   \n",
       "6425093           0.0                1.0             1900.0           0.0   \n",
       "\n",
       "               ...         target_lag_4  target_item_lag_4  target_shop_lag_4  \\\n",
       "6425089        ...                  0.0                0.0                0.0   \n",
       "6425090        ...                  0.0                0.0                0.0   \n",
       "6425091        ...                  0.0                0.0                0.0   \n",
       "6425092        ...                  0.0                0.0                0.0   \n",
       "6425093        ...                  0.0                2.0             1844.0   \n",
       "\n",
       "         target_lag_5  target_item_lag_5  target_shop_lag_5  target_lag_12  \\\n",
       "6425089           0.0                0.0                0.0            0.0   \n",
       "6425090           0.0                0.0                0.0            0.0   \n",
       "6425091           0.0                0.0                0.0            0.0   \n",
       "6425092           0.0                0.0                0.0            0.0   \n",
       "6425093           0.0                4.0             1717.0            0.0   \n",
       "\n",
       "         target_item_lag_12  target_shop_lag_12  item_category_id  \n",
       "6425089                 0.0                 0.0                64  \n",
       "6425090                 0.0                 0.0                64  \n",
       "6425091                 0.0                 0.0                64  \n",
       "6425092                 0.0                 0.0                64  \n",
       "6425093                 0.0                 0.0                57  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_test_data = all_data[all_data['date_block_num']==34].drop(to_drop_cols, axis=1)\n",
    "all_data = all_data[all_data['date_block_num']!=34]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test `date_block_num` is 33\n"
     ]
    }
   ],
   "source": [
    "# Save `date_block_num`, as we can't use them as features, but will need them to split the dataset into parts \n",
    "dates = all_data['date_block_num']\n",
    "\n",
    "last_block = dates.max()\n",
    "print('Test `date_block_num` is %d' % last_block)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates_train = dates[dates <  last_block]\n",
    "dates_test  = dates[dates == last_block]\n",
    "\n",
    "all_data['target'] = all_data['target'].clip(0,20)\n",
    "X_train = all_data.loc[dates <  last_block].drop(to_drop_cols, axis=1)\n",
    "X_test =  all_data.loc[dates == last_block].drop(to_drop_cols, axis=1)\n",
    "\n",
    "y_train = all_data.loc[dates <  last_block, 'target'].values\n",
    "y_test =  all_data.loc[dates == last_block, 'target'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R-squared for linreg is 0.146042\n"
     ]
    }
   ],
   "source": [
    "lr = LinearRegression()\n",
    "lr.fit(X_train.values, y_train)\n",
    "pred_lr = lr.predict(X_test.values)\n",
    "\n",
    "print('Test R-squared for linreg is %f' % r2_score(y_test.clip(0,20), pred_lr.clip(0,20)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.049918661908493\n",
      "1.0493949987899915\n"
     ]
    }
   ],
   "source": [
    "print(np.sqrt((sum((y_test.clip(0,20)-pred_lr.clip(0,20))**2))/(y_test.shape[0])))\n",
    "print(np.sqrt((sum((y_train.clip(0,20)-lr.predict(X_train).clip(0,20))**2))/(y_train.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "Found array with 0 sample(s) (shape=(0, 21)) while a minimum of 1 is required.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-22-d5a45d952cb5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0msub_pred_lr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msub_test_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\base.py\u001b[0m in \u001b[0;36mpredict\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    254\u001b[0m             \u001b[0mReturns\u001b[0m \u001b[0mpredicted\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    255\u001b[0m         \"\"\"\n\u001b[1;32m--> 256\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_decision_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    257\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    258\u001b[0m     \u001b[0m_preprocess_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mstaticmethod\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_preprocess_data\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\base.py\u001b[0m in \u001b[0;36m_decision_function\u001b[1;34m(self, X)\u001b[0m\n\u001b[0;32m    237\u001b[0m         \u001b[0mcheck_is_fitted\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"coef_\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    238\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 239\u001b[1;33m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'csr'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'csc'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'coo'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    240\u001b[0m         return safe_sparse_dot(X, self.coef_.T,\n\u001b[0;32m    241\u001b[0m                                dense_output=True) + self.intercept_\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36mcheck_array\u001b[1;34m(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)\u001b[0m\n\u001b[0;32m    460\u001b[0m                              \u001b[1;34m\" minimum of %d is required%s.\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    461\u001b[0m                              % (n_samples, shape_repr, ensure_min_samples,\n\u001b[1;32m--> 462\u001b[1;33m                                 context))\n\u001b[0m\u001b[0;32m    463\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    464\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mensure_min_features\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0marray\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Found array with 0 sample(s) (shape=(0, 21)) while a minimum of 1 is required."
     ]
    }
   ],
   "source": [
    "sub_pred_lr = lr.predict(sub_test_data.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=4,\n",
       "           max_features='auto', max_leaf_nodes=None,\n",
       "           min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "           min_samples_leaf=1, min_samples_split=2,\n",
       "           min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=-1,\n",
       "           oob_score=False, random_state=0, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "clf = RandomForestRegressor(n_estimators=100, max_depth=4, random_state=0, n_jobs = -1, verbose=1)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_rf = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R-squared for rf is 0.254873\n",
      "0.9807371321492697\n",
      "0.9381757394915309\n"
     ]
    }
   ],
   "source": [
    "print('Test R-squared for rf is %f' % r2_score(y_test.clip(0,20), pred_rf.clip(0,20)))\n",
    "print(np.sqrt((sum((y_test.clip(0,20)-pred_rf.clip(0,20))**2))/(y_test.shape[0])))\n",
    "print(np.sqrt((sum((y_train.clip(0,20)-clf.predict(X_train).clip(0,20))**2))/(y_train.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_pred_rf = clf.predict(sub_test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Benchmark: Mean "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R-squared for rf is -0.000711\n",
      "1.136558396230848\n"
     ]
    }
   ],
   "source": [
    "pred_mean =  np.full((238172,), np.mean(y_train))\n",
    "print('Test R-squared for rf is %f' % r2_score(y_test.clip(0,20), pred_mean.clip(0,20)))\n",
    "print(np.sqrt((sum((y_test.clip(0,20)-pred_mean.clip(0,20))**2))/(y_test.shape[0])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Light gbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R-squared for LightGBM is 0.352241\n"
     ]
    }
   ],
   "source": [
    "lgb_params = {\n",
    "               'feature_fraction': 0.45,\n",
    "               'metric': 'rmse',\n",
    "               'nthread':11, \n",
    "               'min_data_in_leaf': 2**4, \n",
    "               'bagging_fraction': 0.84, \n",
    "               'learning_rate': 0.122, \n",
    "               'objective': 'mse', \n",
    "               'bagging_seed': 2**7, \n",
    "               'num_leaves': 2**6,\n",
    "               'bagging_freq':1,\n",
    "               'verbose':0 \n",
    "              }\n",
    "\n",
    "model = lgb.train(lgb_params, lgb.Dataset(X_train, label=y_train), 100)\n",
    "pred_lgb = model.predict(X_test)\n",
    "\n",
    "print('Test R-squared for LightGBM is %f' % r2_score(y_test.clip(0,20), pred_lgb.clip(0,20)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9144167633545289\n",
      "0.8168128467909939\n"
     ]
    }
   ],
   "source": [
    "print(np.sqrt((sum((y_test.clip(0,20)-pred_lgb.clip(0,20))**2))/(y_test.shape[0])))\n",
    "print(np.sqrt((sum((y_train.clip(0,20)-model.predict(X_train).clip(0,20))**2))/(y_train.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "0.352241\n",
    "0.914416763355\n",
    "0.816812846791"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub_pred_lgb = model.predict(sub_test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Catboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 1.1029154\ttotal: 2.16s\tremaining: 41s\n",
      "1:\tlearn: 1.0294724\ttotal: 3.9s\tremaining: 35.1s\n",
      "2:\tlearn: 0.9857883\ttotal: 5.57s\tremaining: 31.6s\n",
      "3:\tlearn: 0.9552109\ttotal: 7.28s\tremaining: 29.1s\n",
      "4:\tlearn: 0.9371892\ttotal: 9.27s\tremaining: 27.8s\n",
      "5:\tlearn: 0.9186031\ttotal: 11s\tremaining: 25.8s\n",
      "6:\tlearn: 0.9091894\ttotal: 13.1s\tremaining: 24.4s\n",
      "7:\tlearn: 0.9019890\ttotal: 15s\tremaining: 22.5s\n",
      "8:\tlearn: 0.8967270\ttotal: 16.8s\tremaining: 20.5s\n",
      "9:\tlearn: 0.8853466\ttotal: 18.2s\tremaining: 18.2s\n",
      "10:\tlearn: 0.8794866\ttotal: 20s\tremaining: 16.3s\n",
      "11:\tlearn: 0.8753645\ttotal: 22.2s\tremaining: 14.8s\n",
      "12:\tlearn: 0.8714032\ttotal: 23.6s\tremaining: 12.7s\n",
      "13:\tlearn: 0.8694466\ttotal: 25.2s\tremaining: 10.8s\n",
      "14:\tlearn: 0.8680499\ttotal: 26.8s\tremaining: 8.93s\n",
      "15:\tlearn: 0.8651640\ttotal: 28.3s\tremaining: 7.07s\n",
      "16:\tlearn: 0.8641188\ttotal: 30.4s\tremaining: 5.36s\n",
      "17:\tlearn: 0.8623801\ttotal: 31.6s\tremaining: 3.52s\n",
      "18:\tlearn: 0.8524468\ttotal: 33.4s\tremaining: 1.76s\n",
      "19:\tlearn: 0.8503367\ttotal: 34.8s\tremaining: 0us\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostRegressor at 0x17b137964e0>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from catboost import CatBoostRegressor\n",
    "cat_features = [0,1,20]\n",
    "cb = CatBoostRegressor(iterations=20, learning_rate=0.3, depth=3, thread_count=11)\n",
    "cb.fit(X_train, y_train, cat_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R-squared for catboost is 0.172418\n",
      "1.0335773814558504\n"
     ]
    }
   ],
   "source": [
    "pred_cb = cb.predict(X_test)\n",
    "print('Test R-squared for catboost is %f' % r2_score(y_test.clip(0,20), pred_cb.clip(0,20)))\n",
    "print(np.sqrt((sum((y_test.clip(0,20)-pred_cb.clip(0,20))**2))/(y_test.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8364470663085795\n"
     ]
    }
   ],
   "source": [
    "print(np.sqrt((sum((y_train.clip(0,20)-cb.predict(X_train).clip(0,20))**2))/(y_train.shape[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test_level2 = np.c_[sub_pred_lr, sub_pred_lgb] "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train on whole traning data Lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data['target'] = all_data['target'].clip(0,20)\n",
    "X_train = all_data.drop(to_drop_cols, axis=1)\n",
    "\n",
    "\n",
    "y_train = all_data.loc[:, 'target'].values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_preds = model.predict(sub_test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lgb_params = {\n",
    "               'feature_fraction': 0.45,\n",
    "               'metric': 'rmse',\n",
    "               'nthread':11, \n",
    "               'min_data_in_leaf': 2**4, \n",
    "               'bagging_fraction': 0.84, \n",
    "               'learning_rate': 0.122, \n",
    "               'objective': 'mse', \n",
    "               'bagging_seed': 2**7, \n",
    "               'num_leaves': 2**6,\n",
    "               'bagging_freq':1,\n",
    "               'verbose':0 \n",
    "              }\n",
    "\n",
    "model = lgb.train(lgb_params, lgb.Dataset(X_train, label=y_train), 100)\n",
    "sub_pred_lgb = model.predict(sub_test_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train on whole traning data Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = LinearRegression()\n",
    "lr.fit(X_train.values, y_train)\n",
    "pred_lr = lr.predict(X_test.values)\n",
    "sub_pred_lr = lr.predict(sub_test_data.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Second level training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple convex mix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates_train_level2 = dates_train[dates_train.isin([27, 28, 29, 30, 31, 32])]\n",
    "\n",
    "# That is how we get target for the 2nd level dataset\n",
    "y_train_level2 = y_train[dates_train.isin([27, 28, 29, 30, 31, 32])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n"
     ]
    }
   ],
   "source": [
    "# And here we create 2nd level feeature matrix, init it with zeros first\n",
    "X_train_level2 = np.zeros([y_train_level2.shape[0], 2])\n",
    "\n",
    "# Now fill `X_train_level2` with metafeatures\n",
    "for cur_block_num in [27, 28, 29, 30, 31, 32]:\n",
    "    \n",
    "    print(cur_block_num)\n",
    "    \n",
    "    '''\n",
    "        1. Split `X_train` into parts\n",
    "           Remember, that corresponding dates are stored in `dates_train` \n",
    "        2. Fit linear regression \n",
    "        3. Fit LightGBM and put predictions          \n",
    "        4. Store predictions from 2. and 3. in the right place of `X_train_level2`. \n",
    "           You can use `dates_train_level2` for it\n",
    "           Make sure the order of the meta-features is the same as in `X_test_level2`\n",
    "    '''  \n",
    "    M=15\n",
    "    X_train_m = X_train[dates_train.isin(list(range(27-M,cur_block_num)))]\n",
    "    y_train_m = y_train[dates_train.isin(list(range(27-M,cur_block_num)))]\n",
    "    X_test_m = X_train[dates_train==cur_block_num]\n",
    "    lr = LinearRegression()\n",
    "    lr.fit(X_train_m.values, y_train_m)\n",
    "    pred_lr_train = lr.predict(X_test_m.values)\n",
    "    \n",
    "    lgb_params = {\n",
    "               'feature_fraction': 0.45,\n",
    "               'metric': 'rmse',\n",
    "               'nthread':11, \n",
    "               'min_data_in_leaf': 2**4, \n",
    "               'bagging_fraction': 0.84, \n",
    "               'learning_rate': 0.122, \n",
    "               'objective': 'mse', \n",
    "               'bagging_seed': 2**7, \n",
    "               'num_leaves': 2**4,\n",
    "               'bagging_freq':1,\n",
    "               'verbose':0 \n",
    "              }\n",
    "\n",
    "    model = lgb.train(lgb_params, lgb.Dataset(X_train_m, label=y_train_m), 100)\n",
    "    pred_lgb_train = model.predict(X_test_m)\n",
    "    \n",
    "    index1 = dates_train.isin(list(range(27,cur_block_num+1))).sum()- (dates_train==cur_block_num).sum()\n",
    "    index2 = dates_train.isin(list(range(27,cur_block_num+1))).sum()\n",
    "    X_train_level2[index1:index2,0] = pred_lr_train\n",
    "    X_train_level2[index1:index2,1] = pred_lgb_train\n",
    "    #  YOUR CODE GOES HERE\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_level2 = X_train_level2.clip(0,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3XlwXNd96Pnv797b+4rGvhAAN3EVF4mSKFm7ZMmyHW/ynsRL8iJPKk5eal5NtqmaZOY912Sq5iXjqbw4I5ft2Ilt2bHiWLblRZas3aTETdxJgSv2Heh9ufee+aObC0iIAEiAAIjzqUIROH379q9B9K9P/86554hSCk3TNO3GZ8x3AJqmadr1oRO+pmnaEqETvqZp2hKhE76madoSoRO+pmnaEqETvqZp2hKhE76madoSoRO+pmnaEqETvqZp2hJhzXcAADU1Naq9vX2+w9A0TVtUdu/ePaSUqp3u8Qsi4be3t7Nr1675DkPTNG1REZEzMzlel3Q0TdOWCJ3wNU3Tlgid8DVN05YInfA1TdOWCJ3wNe06UaqE4/TgOiOzfm5XFSjZ3TjO6Kycr+iWGMyPkLazs3K+6yFrZxguDFN0i/MdyoK1IGbpaNqNrlTcTyH7fVA5FC6WZw2+4KcxjOg1nzuT30ky+x+4qgi4BLybiIc+iWEEZnwupRT7x4/z2uAebOWgUKyJLOfB+jvwGp5rjnUulNwSO4dfpyN9HEEQMbm1ahvroxsRkfkOb0HRPXxNm2OO00Mh88+I+DDMJgyjGbvUQT7zr1zrjnOF0gnGMt9BjDAeqwnLbCJXfIuxzA+u6nxns7280L+DsBWg1ldFjTfO0eRJXhncfU1xzqXdo29yPHWMuCdBlbeasBlmx/DrdOXOzndoC45O+Jo2x+zCmyAGIkEARATDaMCx30a5g9d07kz+NUQCGOKvnNvAMpvJFffiuKkZn++tsaMETD+eSm/eEIMaXxWHkycoOAuvVFJySxxPHaXKm8CQcjqzDIuAGeTQ+MF5jm7hmbKkIyJ+4GXAVzn+B0qpvxaRfwbuA8Yrh35OKbVPyp+hvgy8F8hW2vfMRfCaNl+G+sZ488UjdJ0YJFEX5fYH17FsZf2kxyp3FPBOaCu/TAyUyl1THI4aQ8R3ybkNECrnjszofGk7h9eYmBZMMVBKUXRL+EzvO9xzfpTcEq5yMS7pu3rEQ9ZZPOMP18t0evgF4EGl1GZgC/AeEdleue1/UUptqXztq7Q9BqyufD0BfGW2g9a0+TTUN8Z3/t/neHt/J6Yl9HUO8b1/fJ63D3ZOerzhWY9SE5OPUnlEPBjm5G8S0xXwbMR1xye0uW4GUyKYRvWMz7ci1ELqkoHajJ0j6gkTsmY+JjDXAmaAmCdG7pLknnFStAbb5imqhWvKhK/K0pUfPZWvKxUePwh8q3K/HUBcRBqvPVRNWxjeeOEwSrkk6qJ4fR5iiTDRqiAv/3gfrutedrzHuxnTXI7jnMV1R3Gdflx3EG/gw0ilFHO1gr7tWGYjJfssjjuK7fThqjFiwY8hYs74fJviNxHzROjPj5AqZRgujpFz8jxQd/v5kslCIiLcWXM3RVVgrDRCxs4wXBwkZIVZH90w3+EtONOapSPlv5zdwCrgfyildorIHwJfEpH/DXge+AulVAFoBi7u6nRV2npnNXJNmyddJweJxIMT2oJhP/3doxTzJfzBS0ssPgKRL1Aq7sMpHUCMCB7vdkyr/ZpjMYwgtdE/JlvYTaF0BNNMEPLdicdqvqrzBa0An2h9jKPJk5zN9BL3RtgQW0WNr+qaY50rDf5GPtD0EY6njjJeGqfR38SqyGr85sL7RDLfppXwlVIOsEVE4sAPRWQj8JdAH+Xi5JPAnwP/BzDZPKjLPhGIyBOUSz60trZeVfCaNh8StVEGekbw+i5MUywWSvj9Xjy+yacuivjx+raDb/ukt18LwwgSDtxDOHDPrJwvYPrYWrWOrVXrZuV810PcW8Xt1XfOdxgL3ow+oymlxoAXgfcopXorZZsC8A3g9sphXcCyi+7WAvRMcq4nlVLblFLbamunvbqnps272x9aTzZVIJcpAOVkP9yX5I6HN2CaC6/soWnnTPnXKSK1lZ49IhIAHgaOnqvLV2blfAg4NwfqGeAzUrYdGFdK6XKOdsNoXVXPBz9/D2IIA92j5NJFHvjQrdxyz03zHZqmXdF0SjqNwDcrdXwD+L5S6ici8oKI1FIu4ewD/qfK8c9SnpLZQXla5udnP2xNm1+rb17Gyg3NFPMlPD6P7tlri8KUCV8ptR/YOkn7g+9wvAL+6NpD07SFzTCMywZoNW0h090STdO0JUInfE3TtCVCJ3xN07QlQid8TdO0JUInfE3TtCVCJ3xN07QlQid8TdO0JUInfE3TtCVCJ3xN07QlQid8TdO0JUInfE3TtCVCJ3xN07QlQid8TdO0JUInfE3TtCVCJ3xN07QlQid8TdO0JUInfE3TtCVCJ3xN07QlQid8TdO0JWLKhC8ifhF5Q0TeEpFDIvK/V9qXi8hOEXlbRL4nIt5Ku6/yc0fl9va5fQqapmnadEy5iTlQAB5USqVFxAO8KiI/A/5n4O+VUk+JyD8Bvw98pfLvqFJqlYh8Evi/gE/MUfyaNudcV3Gqb5j9e08yemqQluo4m+5cTUN73XyHpmkzMmXCV0opIF350VP5UsCDwKcr7d8E/oZywv9g5XuAHwD/ICJSOY+mLSqFks23f72Hl14+yPhwCtNjEnSEtc8f4Lc+ex+b7lk33yFq2rRNq4YvIqaI7AMGgOeAE8CYUsquHNIFNFe+bwY6ASq3jwPVsxm0pl0vvzl6hn1Hz2IPp2mIhakNBnFDHoYSXp7/7uvkM/l3vG/BKXE82cP+0bMMF1LXMWpNm9x0SjoopRxgi4jEgR8Ck3VrzvXg5Qq3nSciTwBPALS2tk4rWE273nZ1dCHpEpZlce5POyQmA8qmyYGBzmGWrWniTHKMff29OEqxqbaesN/k26dfJeMUzv/131u3lvvrNyAy2UtkdrjKZrxwjGTxbSwjRMK/iYBVP2ePdymlFN25Xk5mzmBgsCqynHpf7Zw+Z236ppXwz1FKjYnIi8B2IC4iVqUX3wL0VA7rApYBXSJiATFgZJJzPQk8CbBt2zZd7tEWLI/XRLnuZe1KKbx+D8+fOcnPThzDMgxEhFc7T2OHRmmNRWn0xwFwlMtLA0doD9exPDw3tX9XlTg5/hTjxWOY4kdh0599leXRj1Pl3zAnj3kxpRSvDu3kwPhhPGKhUOwfP8z26lu5pWrTnD++NrXpzNKprfTsEZEA8DBwBPg18NHKYZ8FflT5/pnKz1Ruf0HX77XFatuqFoj6cZXCLpUrmGnXIZZzqGuuxqoN8YuTb9MQjtAQjlAfClMV8nIqOYS45vnzmGLgNSwOjXXOWaxjhWOMF48SNJvwm9UEzHq8RoyzqR/hqtKcPe45g4VhDo4focaboMobJ+GtotobZ+fIbpIlXdJaCKbTw28EvikiJuU3iO8rpX4iIoeBp0TkvwF7ga9Vjv8a8C8i0kG5Z//JOYhb096Ro4oM5nYwktsNCDWBbVQHbscszxyekTvXtnF6YJRSrkj3iT6cXIGgErY2NvChP3yEU+kkCoVlXOg7iYAgjOZzRH2+C+2AM4d9n/HCESwJTiifWEaAkj1Ozu4n5GmZs8cG6Mv3A2DIhd+FKSYoGCgMEfVE5vTxtalNZ5bOfmDrJO0ngdsnac8DH5uV6DRthpRyOTX+HVLF43iNakDRlX6WVPEUK2K/M+Nass9j8ZkHbuX+jSvoH0lhp/KsbKimflk1IoJv6PJB26DhxxST0kW9ale55J0SG+Jzl3QtI4iLPaFNKYVCYVzFm91MeYx3fgyPzKh6rM0R/b+g3VAypTOkim8TMJvPJ3dTAiSLx8jYZwl72mZ8TsMQ2usTtNbFSZbGJ/RgV1YlCHu9jBfyxHx+ALIlmzarhbDPoTc3xrlR29trVrFijur3AAn/FgZzO3BUAVN8KKUouMOErGb8Zu2EY5VSjGfzOK5LVSiIYVz7oGprsBlLTHJOnoBZ/l2k7QwBM0BToOGaz69dO53wtRtK3hkCmNCTP/d9wR66qoQPMFjo55XBX5OykwDUeGu5p/ZBop4Yv79pG988sIfuVBIDIeDx8MUtd9MUjXAi1U/eKdASrKExEJ/T2SohTzOtkQ/Rlf4pReWgUAStZpbHPjHhcUfTOX644yCnB0ZAoDoc5PHtG6mLnqJUfAnlZrA8W/D678UwotN/fCvIY40P86v+lxgqjoCCsCfEo/UP4DE8c/GUtRmShTCeum3bNrVr1675DkO7AYwXjnFy/F8JWo0T2rN2DyvjnyHqvWnG58w5OX7Y9T1MMQlaIZRSpO0kPjPAh5o/hikmjuvSnU7iKkVzOIrHNKc+8Ryx3Rw5ux/T8BEwGyYke8d1+cef/YbRdI6aSLnen8zmaY6/wWObz+K1qhHx4rrDGEYtocifIkZwho9vM1QcQRBqfdUTPhFps0tEdiultk33eP0/oU2Lq1wG8gOczZ4lbaenvsM8iXhX4DdryDv9KOWilEPOGcBv1RP2rLiqc3Zmz1B0iwStEFD+xBDxxEiXkvTn+wAwDYPWaJz2WNW8JnsoD9RGvO0ErcbLPlF0Do0zMJ6mNho6f1si7NIS28NwOoZhxBAJYJotuO4gpeK+q3h8iwZ/HfX+Wp3sFxhd0tGmlLEzPNf3HEPFofNJYnNsM7dW3brgLqgxxMPK+OfoTj/LeOEwAHHfRprCj2Fc5cBh3slN+jwVUHQL1xLudZcrFJFLro30W+OICMXSxHaRAI5zArjrOkaozSWd8LUpvTL4CqOlUWp8NUC5t79ndA+1vlraQldXE59LXjPG8tincFQR4KqmY16szl+Pq1yUUucTv6tcRCDhXVyrhjRURQGF47qYlamkBTuAwiUW8k04Vqk8hqEXiLuR6M9b2hVl7Szd+W7invj5NkMMAmaAo6mj8xjZ1EzxXnOyB6jzNbAitJLh4iBpO0WqlGSkOMTG6GaintgsRHr9VIUD3L1+Ob2jKUbSWcazeU4NCmJsJOofRikbpRSuO4KIB4/3tvkOWZtFuoevXZFdSQCXMsSg5M791ZsLgSEGd9c+QGuwnZOZE1hisjJ8E82BZfMd2lV59+bVtNVWsetEFyXb4ZEtN7Gx7R6c4s8oFXYADqbZij/4OIaZmO9wtVmkE752RRErQtwTJ+tkCVUGLZVSZOwMt1TdMs/RXT+mmCwPr2J5eNV8h3LNRIQ1zbWsaZ44Nx/ro/gDHwBVgkuu2NVuDLqko12RiHBv7b04ymG4OMxocZSh4hAtwRZW3QDJ70ailMKxz1Is7MQuHURVxjBmQsSLGCGd7G9QuoevTanOX8dHWz7KqcwpMnaGhkADzYHm8jop2oKglE0++11Kxd2U5w8JhpEgGP5DjEuustWWLp3wtWkJWkE2xOZ+iV3t6pSKeykV3sAw2y7MJHIHyGW/RyjyxXmOTlsodElH0+aA47gMdo0w1DM66aD3TChVQrnpK56nVNyJGFWXLClRi2OfwHXHrunxtRuH7uFr2izrOTXAT7/+IsmRDADVjXHe/3v3U9NUNaPzKGVTyj+HXXgJpUoYRg2ewIexvGsnO/raA9dueLqHr2mzKJvK8/Q//BLbdqhblqC2pYr0WIan/8cvKRXtqU9wkVLuZ5TyPwepwjCbUapIMfNVHPvyTVQ83jtQauKnCaWGMK3liCyuawW0uaMTvqbNolOHOinmCoRj5QXHRIRYTYT0WIbO473TPo9SOUrFVxCjGZHySpNiRFBY2IVXLjve470Fj3cbrtuF43TiOJ2IBAgEP6ln3Gjn6ZKOps2iXKZQ3vJqEsX89C9UU24GlIsYE1+iIiFcd+Cy40Us/MHP4PXdh+P0IhLG8tyEiO+yYxej8tW/fSg3i2E2YsxwBU+tTCd8TZtFTctrUa7CddX5TUUcp7wBen3r9NfdESOGiB+l8oj4L9ygxjHNyS94ExFMqx3Tar/q+Bci102SSX8T2z4BYiAY+AMfwO+/d75DW3R0wte0WdS4vI4Nd67mwGvHCYT9KKXIZwtsf88WquqmX0sX8eAJfIBi9jsoiZSTvjsGEsLyv2sOn8HCk808hW2fxqjsYqZUkVz2B5hmIx7Palx3lGJxL647hmWtwuNZd74Mpk00ZcIXkWXAt4AGwAWeVEp9WUT+BvgDYLBy6F8ppZ6t3Ocvgd8HHOBPlFK/mIPYNW3BEREe+e27WbW5jaNvnsS0DNbfvoq2dU0zPpfHdzuGEaGUfwnljmD67sTy3YdhzGy2z2LmuqOUSofPJ3soXw2MBCgWdiJikk79E4oSYFHIv4jlWU04/MQNU86aTdPp4dvAf1FK7RGRCLBbRJ6r3Pb3Sqn/++KDRWQ98ElgA9AE/EpEblJKObMZuKYtVKZpsHpzG6s3X/vS0aZnHaZn3SxEtTgpVR4TuXTgWcSD46bIZL4D4sOsLOOslMIuvU2hsAv/EvskNB1TztJRSvUqpfZUvk8BR4DmK9zlg8BTSqmCUuoU0AHcPhvBapq2tBhGDYZEcN0Lu6wppVBuEo/VXtmK8UKpTEQQI0aptGc+wl3wZjQtU0Taga3AzkrTF0Vkv4h8XUTOfc5sBi6eKNzFJG8QIvKEiOwSkV2Dg4OX3qxpmoaIRSD4KZRK4di9OM4QrtOFaa3A67sFFJdfgaxsRALzE/ACN+2ELyJh4GngT5VSSeArwEpgC9AL/Pdzh05y98suA1RKPamU2qaU2lZbqxd30jRtcl7vOqLRP8PnfxCv52aCoc8QifwhplmPx7MW1+k7n/SVsnFVGp9Pb8s4mWnN0pHykPfTwLeVUv8OoJTqv+j2rwI/qfzYBVy8M0QL0DMr0WraEqOUYmg0Q6FYoqYqjN+3NGefmGY9weD7LmsPhj5JOv11HOdsZZFQIRB4H5a1dMc9rmQ6s3QE+BpwRCn1dxe1Nyqlzl06+GHgYOX7Z4DviMjfUR60XQ28MatRa9oSkM4W+OEv3+JM9whigGkYPPKutdyysXW+Q1swDCNGJPKnOE4XSmUwzcYJNX1toun08N8F/C5wQET2Vdr+CviUiGyh/L56GvgCgFLqkIh8HzhMeYbPH+kZOpo2cz9+4QCdvSPU10QQEYolm5++dIiaRJjWJr314DkigmUtzu0mr7cpE75S6lUmr8s/e4X7fAn40jXEpWlL2ngqx4mzQ9RXR85PSfR6LLwei72Hu3TC166KXjxN0xagQtFGhMvmn3ssg0xu5lsXahrohK9pC1IiHiLo95LNX0juSinS2QJrV9TNY2TaYqYTvqYtQJZp8P4HNpLKFBgYTjE6nqV3IElrU4INq2e+TIOmgV48bclwbIdTHQN0nh4kHPGzZkML0bheYnYhW91exxOfeBf7j3WTTOdZ2VrD2hX1eD36ZatdHf2XswSUSjb/8d0dnD4xgMdj4Tgur714lMd/+y6WtdfMd3jaFdQmwjx055r5DkO7QeiSzhJw9GAXpzoGqG+MU10boa4hht/v4ec/2oPruvMdnqZp14lO+EvAsYPdhMP+CTM+QmE/ybEso5WNthcyV7kUXfvyNVM0TZsRXdJZArw+z/ldl845lzw9ljkfIU2Lq1zeGH6b1wePkHWK1PtjvLthC+3h+vkOTdMWJd3DXwI23dJGLlfEti9c8Dw8lGLZ8poFPXD7+uBRftG7F7/ppcEfJ23n+c6Zl+nNjcx3aJq2KOmEvwS0razj3oc3MDKUZqBvnP7eMWpqo7zng5PvjboQFF2b14eOUu+P4TM9iAhRTxBLDHYOH5/v8DRtUdIlnSVARNh+7xo2bm1lsD9JIOClrjGGYSzc9/ucXcBWDh5j4p9o0PQxkB+fp6g0bXHTCX8JCUcChCOLY2OIkOXHb3opOCV85oUlgVN2ni2RxiveN2vn6MsPYYlJY6AWj7E0lxS+Etd1yWUKeH0ePF6dBpYK/T+tLUiWYfJA3c38uPsNIp4AAdPLeCmLJQZ3VN/0jvc7nHybFwd24qryILXf9PP+pgdp8OvrDc45dbiLF/7tDcaHU1iWyS0PrOfOxzZjLuABfG12LNzP9NqSt6VqOZ9ovZu4J0TOKXJTpInPrXiYhC8y6fEjxTGe7/8NUStErS9BrS+BgfDTnhcoufZ1jn5h6jszxA//6Xkc26auJUE0EeI3P3uLV3+8d75D064D3cPXFiwRYU2shTWxlmkdfzJ9FgOZUMIJWQEGCyP05wdpCV65FLQU7H3pCJbHJFgp7Vlei7qWKva+fITt79mEL+Cd5wi1uaR7+NoNw3YdZLKdGwBH6SuKAYb7xvCHfBPaTMvEdVxy6fw8RaVdLzrhazeMtlAztnLP1+8Bim4JU0wa/LXzGNnC0bamicx4dkJbIVfEH/QRjofmKSrtetElHW3WuapI0T6Ncm3GBiJkUwaJ2ihVtZPX3mdLg7+WW+Ib2Dt2CEMMFApBeKT+HnzmwitV5OwSp1OjmCK0RarwmXP/ctx8zxoO7uhgsGeUSDxEMV8kk8zz3s/eg+XRg7Y3OplqfRIRWQZ8C2gAXOBJpdSXRSQBfA9op7yn7ceVUqOVTc+/DLwXyAKfU0rtudJjbNu2Te3atesan4q2EBTs0wylvkaxmKSzo5/0eInuQ7eT7G9l852reOBDt2KaFz5YOrbD2eO9DHSNEE2EWbGh5ZrqyEop+gvDdGV7sAyL5cFlxLxz+0ZzNfYP9fLdjn3YjosSCJoePr9uG8ujc7914fhwmt2/PsTpwz3EqsNse2gDbWv1GvuLkYjsVkptm/bx00j4jUCjUmqPiESA3cCHgM8BI0qpvxWRvwCqlFJ/LiLvBf6YcsK/A/iyUuqOKz2GTvg3BlcV6B39r4iYnDqcZLB3jHDMwPKn6XnrI3SfsHn0k9vZdMdKoFxK+I8nn6fzeB9iCEopIvEQH/vjR0nUx+b52cydkXyWv93zInGfn4BVHmBOFQuUXJf/dduD+K9DT1+7Mcw04U9Zw1dK9Z7roSulUsARoBn4IPDNymHfpPwmQKX9W6psBxCvvGloN7hC6QSuyoAbZqh3jGDYh3K9oBTBRBfRRIj9v3n7/PH7Xj3K2eO91C1LUNeSoH5ZNYVckef/bcc8Pou5d2R0AFe555M9QMTrI2cXOTk+PI+RaTe6GQ3aikg7sBXYCdQrpXqh/KYAnNtosxnovOhuXZU27YbnAOWeuuKiDbhFELExTaFYuDAf/vDOE8QS4QnLNsdrI5w91kM+W7i+oV9HtuvCZLOJBGw9m0ibQ9NO+CISBp4G/lQplbzSoZO0XVY3EpEnRGSXiOwaHBycbhjaAua12hExMCybaFWIfLYIuAguhVQT4yMZ1t/afv540zIvW+NeueWB1oW8zs90FJwC/flBkqXUZbetjFWjFDgXbT5TdGwMyoO3mjZXplUsFBEP5WT/baXUv1ea+0WkUSnVWynZDFTau4BlF929Bei59JxKqSeBJ6Fcw7/K+LUFxDQiVAU/xkjmKZatgdNHx3GcEt1H19NzzKCxtZotd60+f/zmd93EL779Gv6QH8Mo9xOG+8a56ZZ2vP7Fuf6NUoq3xg7yxsheFApXuawML+f+urvwGuXB6OZQlIdaVvF8VwdmZTaRUorHV95MzOu/6sdOjmdxbJdYVXDRv2Fqc2PKhF+ZdfM14IhS6u8uuukZ4LPA31b+/dFF7V8UkacoD9qOnyv9aDe+kP8OvJ42csGDxG/N0dMRJWhH2XZHNcvXNk1YqGvjnavpPjnA4TdPlMs6SlHfWsMDj98+j8/g2pzOdPLa8JtUe+JYhoVSihPpU/gML/fV3QWUS12Pta5hY6KBo2MDGAgbqxtoCF7dbKLkeJZf/GgPZ04MgkC8KsRjH7mV5mXVs/nUtBvAdGbp3A28AhygPC0T4K8o1/G/D7QCZ4GPKaVGKm8Q/wC8h/K0zM8rpa44BUfP0pkfju3gKoXHM3+zQpRSDPWOMdI3RigapGlF7aLunT7T8wuG88NEPOHzbY5yGCsl+fzyT53v5c8W13X51ydfZHgwRXVtBBEhncpRLDj83h8/TCS6OFZHXQwKRZsTnYOMJXPUVkdY3lyNZc7v3+pMZ+lM+UpXSr3K5HV5gIcmOV4BfzTdALTrr5Av8fqLR3hr12kc26F9VT33P7qR6trodY9FRKhtqqK26caoXefs3GVr+BsYKKUoufasJ/y+7jEGesepb4qfbwtHAgykxzh+qJtb71w1q4+3VI2lsvzLj99kNJlFpDwxYVlDFZ967Fb8vsVTfly8XSntqiilePaHu9n1egexqiA19TG6zgzxvW+8SuYKa6mMj6Q5/tZZzhzrxS4573jcYpIsZTid6WW4MD5rG6SvCLeRciZuDJ91csS9MYLm7Pe28/nipOsHGaZBKpmb9cdbqn75+lFSmTyNtTEaaqI01ETp7B1l54HT8x3ajOgrPJaYkaEUHUd7qW+Kn58OmaiJ0N9b7hFurVwUdY5Sih2/PMDrv9iPVKZcRqtCfOSJB6luWJwXRznK5aWBPewZPY6B4KJYEW7ifY13XfMSDBuia+lIn2KwMIzf8FFUJQTh3Q33T5h+Oltq62MowHHc81cwK6WwSw7Lluv1g2ZDyXY4dnqAusSFMRYRoSoW5K1j3dy3bfUV7r2w6B7+EpNK5jEMuSz5eDwmw0OXTyHs7Ojn1WffoqY+Rl1zFfUtCYr5Ej/51iuz1iu+3g6Nn+LNkSPU+mLU+uPU+eKcSHfzyuBb13zuoBXgw83v457a7TQFG9kc38jHl32QpkDDLER+uUg0wJ33rqG/Z4yx0QypZI7e7lHaV9XTvkIn/NkggCGCumR2uVIKc5GNN+ke/hJTlQihXIXruhMGR0tFh4ZJ6uhHdp/G5/dM2A0pmggx2DPGUO/Yoqy97x09RswTxpDy8xcRan0xDoyf4P66W7CMa1tEzG/62Bhbx8bYutkId0p3PbCOxmUJDuw+Q6FQ4u4H17NuU4vewWqWWJbJxtWNHDjeQ31NeZxLKcVIMst77ro+/8ezRSf8JSZWFWLLbcvZvfME8UQIyzQYG8mQqAmzet3lC2g5tnONWMi5AAAgAElEQVR+jvw55z4duO7i7OEX3CKWTOyZGRg4ysFRDhaLK1GKCCtWN7Bi9dx8ipiKUnncwpuo0kEwwhje7Yi1ak5KWPPl4TvWMDSWoad/DCqDthtWNrJtQ+t8hzYjOuEvQQ88toma+ih7dp6gkLe59c5V3HbXanyTXOx005ZWDr15gqgbPp/4M6kcoWiAmkVaw18baeONkcPUmxdWphwrpWkNNizIZZQXMqUKOOknUfZZMKLg9OAU92EEPoTpv2e+w5s1oaCPz39wO139oyQzBapjIRpqIovuTU0n/CXINA223LaCLbetmPLY5eua2Xj7Kg69eQLDKE8v9PgsPvLEg4u2ZHBrYh0n0t305UbwmRYl18Fvenmg/tb5Dm3RcYsHUPYZxLpwcb1SEdz8sxjeWxEjOI/RzS7DEFob53756rmkE752RaZp8OintnPznavoPjlAIORjxfpmwrHF+0IOWX4+3fYIHekuenJDJLwx1kZbCVn6IqWZUvYxkIl/CyJeFC7K7UeM5fMUmTYZnfC1KRmGQcuKOlpW1E198CLhM71siK1gQ2zqTznaOxMjgaI4oa08e8tFRG+ZuNDohK9NW/fgOHuOdZFM51jdWsemlY34fR6UKpErHaVQ6sA04gS9W7DMxTd7R5s5w3srbuEllJtGjDBKueD0IZ6bwNDTQhcanfC1aTl4spenX3gLj2ni9Zq83TnE3mNd/O5jm0gXvkW+dBwRH4oS49mfUxf9Aj6P7j3f6MSswwh+Hjf3A5RTXhRXPBswg48vugHNpUAnfG1Ktu3w09cOUxUJnF83JBry0zOY5NDp56irOYbXbD3/AnfcJMPp79IY/0tEFteFKdrMmd41GJ6/AHcExIcY139NJm169KtRm9JwMkuhaF+2SFQ46GU4uQvTiE3ozZlGFNsZxnb1dn1LhYiJmLU62S9wOuFrU/J7LRTgXrKUQqFk4/OGQNkT2s8tuSD6A6SmLSj6FalNKRYOsLatjmNnBqhLhDFEyBdLlGyX5Q0Pky98k4HeIuOjBfwBi4Zmh+qqDXrgVtMWGJ3wtWn5rbs3gIKjZwcQwOe1ePyBTcS8UZ79WStVjbvx+EwKJYdD+6rYsPpe6q7hQtwzmX7eGD7CWClNW6ie2xJrqfJe3Y5Qsy1VSnIoeZC+XA9Rb4wN0Zup98/PsgaaNhM64WvTEvR7+cS7tzKezpEr2FTHgngsk+d/cYC+0+tRpU14/cM4tp/kaJQXT5/lprVrzi/ZOxOHx0/zTPdrBEwfftPL/tETHE2e5TPtjxL3hqc+wRxKlpL8tOc/KLpFgmaInmw3p9OneKj+EdpC7fMam6ZNRdfwtRmJhQM0VEfwVJZVOHNqkEjUj2sHyaeXUcrXEgj4yKQLZDOFGZ/fUS6/HthHlTdC3BvGb3qp9ccpOCV2jxyf7aczY4fGD1BSJaq81fhMP1FPjLAV5o2R3+Aqd+oTaNo80glfuyaJRJh8rjShzS45WJYx6WJsU8nYObJ2Dv8li5hFPAHOZPuuKdbZ0JPrJmhOvILUbwbI2BnyzjvvGKZpC8GUCV9Evi4iAyJy8KK2vxGRbhHZV/l670W3/aWIdIjIMRF5dK4C1+aPq1yOJk/z/c5f0dfaS29ymEymnOzsksPgQJJb71iJ1zvziqHf9GGKie1O3EYx7xRJeOd/yl/UE6XgTFxKwHZtLLFmfb9aTZtt0+nh/zPwnkna/14ptaXy9SyAiKwHPglsqNznH0VkcS6pqL2jF/p38eOeVxjOj0ONg/muAifSXfT3jZFM5njXfWu56541V3Vur2GxLbGGwcIYJbc83TNr5yk4JW5LXN05Z9PG2CYKbp6CUy5X2a7NWGmE9dGNWIYeEtMWtin/QpVSL4tI+zTP90HgKaVUATglIh3A7cBvrjpC7TJKKc4e7eHga8fIZfKsvmU5625fhdc38xLKTA0Xxnlr7G3qfYnzO0atWdtEX9sw76/eyqpEMx5P+T1+rJDDVi7VvuCMLrN/V81GDDF4c/goJWUT9YT4yLJ7aQ7O/9osjYEmHqh7iDdGdjBSHMYUk83xW9hctXW+Q9O0KV1Ll+SLIvIZYBfwX5RSo0AzsOOiY7oqbZcRkSeAJwBaWxfXrjHzbffzB3nhqdfxh7xYHouTBzo58kYHH/2Tx7A8c9vLHCqMgXA+2UN5xyXTMElZaTwek7FCju+f3EfH+DAI1PpDfGLlFlrD05uXbxomd9fezB3V6yi6NgHTO+Hx5tvy8EraQsvJOVm8hg+PMfdvtJo2G672VfQVYCWwBegF/nulfbJu3KT74CmlnlRKbVNKbautnf+e22KRTeV4+emd1DZXUVUXI1IVoqG9hs6jPZzYf3bOHz9g+SZtd1GELT+uUnzj2BucTo3QFIzQFIiQtYt89ehOUsWZDWp6DIuQ5V9Qyf4cQwxCVlgne21RuapXklKqXynlKKVc4KuUyzZQ7tEvu+jQFqDn2kK8cSmlKNg9jOV+zXjuFUrO0JT3GeoeQSmwLhoQFRF8AS9njnTPZbgANAfqqPbGGC6Oo5RCKcV4KU3I8rMi3MyZ9Ci92ST1gfL2byJC3BugYJc4ODL/s2w0bSm7qs//ItKolOqt/Phh4NwMnmeA74jI3wFNwGrgjWuO8gaklGIs9ytGsz+HSn17JPsMNaFPEfHf8o7384d8KOWilJpQF7dLDpGq2d9wQilF2i5gihC0fJhi8JGWB/hl3w7OZPsQoM5fzaMN2/GbPrL26KT1elMMxoq5CW29A+PsOdzJWDLPqrYaNq1pJnAVUzk1TZueKRO+iHwXuB+oEZEu4K+B+0VkC+VyzWngCwBKqUMi8n3gMGADf6SUciY771JXdHoYzf4cr9mASPm/wVUFhjLfI+i9CdOY/IrS2pZqmlbU0392iOrGOCJCNpVDRFh326pZjbE/l+SHZ/dwsLOT0cEUNfkgj6+4le3b1/LRZQ+RsXO4yiVsXRiUbQxGUYDjuphG+QOkUoqSclgerT5/7mMn+/m3n+/FMg28HouTnYPsOdTJZz9yB0G/nt6oaXNhOrN0PjVJ89eucPyXgC9dS1BLQa50HETOJ3sAQ3wo5ZC3TxPybpz0fiLCB77wMD//5kucPtyNCIRjIT78xUeJ183ePPWsXeQbHa9y6mQ/qbMpfH4PfZ4U/9+Bl+g+MsjH/9N9k+4Bm/AFeaBxJc91v03E48UUg/FinnVVdayO1gBgOy4/e/kwsbCfQCW5R8N+egfH2Xeki7u26o1TNG0u6InD8+Qdlw4WQaYYWgnHQ3z0P7+X5HCKUtEmXhe74po1BafEiXQvo8U0Nd4YYTuEgUFdVfh8L/xSx8f7GE6nyXRliEQD5XECPIxXFTj8djdnOvpZsbZp0vu+Z9la2iIJdg6coeg4PNqyhq01LecfazyVI5srUlc9cTG0SMjP8VMDOuFr2hzRCX+eBL3rGMk+g6sKGFKe+eK4aQzx4beml/Ci1VOvHjlWzPDtMy8yVkyTT7t0vJXCLHhYHm6gKhTk8fs20VZ/+XTJsVKOUr6ECBNr8gKuV9F7duQdE76IsL6qnvVV9ZPefn59fVdhGBfOXSzaRMP+KZ+TpmlXZ+HNd1siPGYNNaFPYLujFOxuCk43LgXqwp/DMGYv6T3ft490KUetJ073/hIex4sK2Nj+Akopvv3cblLZyxc5aw7GEcuYsOmJqsywDeRMolXBq44pFPSxcXUjA8PJ8+cvFG3yRZttG/U1GZo2V3QPfx5F/NsIeteQt08jGPitlbOW7EuuzSsDx/hB5w68YhHOBmkInWJj6ykso0B3soWo837ODrgc7xzk1jUtE+6/IlLLzQ0t/LJzhNFUFn/QQ96wqRsKUO0Ns3LdpNfTTdt77l2H67oc7uhDRPB4TD748CZamxLXdF5N096ZTvjzzDQihLw3z/p5f9S1h70jp7HEwGd6aPK9wfJlJ0GqKdhe2qpOEjefpnf4veQKxcvub4rB76y4k+WeGn70xpuM9qdoGw+zObGMR/7gNoLhyS/Ami6f18OHH9nCQ3flyRdKVFXW19c0be7ohH8DGswnOTB6lmXBBEU3R7LUw/rqLs6eiRL0m4hA0GnE8iVJRE7SWn/vpOfxmhYPtK/jgfZ15LNFHMclGPbNaF2cqUTDfl2317TrRNfw51nRdija5UsVXKUYL+TJ2/YU9yqz3SwFZxR1ycYbY8UsRuUq17ZQA3UeF8dwMaNCLl/Ccj1YrkUqC+vbbZbVxad8LH/QSyjin9Vkr2na9aV7+PNkLJvn2YNHOdgzAEBdVZAxT560U8QQ4V2NbTzWvgaveXmZI10aYt/wVxnNH8AQk4SvnbWxTwODpAp7cGyXiOmi1CY8hsXy8FoS1m6KCT/NdbV48l6UUtTFFVVVmziW7McUoS1UjdfUfxKadqPSr+7r7O3hYV46dYpfHezAFKiOBsi5Nq8fPkN1OMhd61txleLFrpPYyuXxVRcuwCq6BTozR9kz+P8Ao/iNGlyE3txxisU/pc6/Cr/ViFccbg510JEdpmTejyNR+ortNPlPszZegyU+HHeQkYKPr5wYI+/uQClFwPLyuyu30x6ufucnoGnaoqUT/nX0ZncXTx04QLFgM5hNkTRymHkDj2GSw0alFSPJHDJukzmZ4unX3qT2IS93bF1BQcZ5ZfCHDOU7EKcTJISjUkQ9CWzXIFkapjcVxWsGWRFLsDKyGZ95lDdSg2TtIA3xx7kp0k028xpQxBNYx9P9UQJWFVVmef2adKnAv5zYwZ9tfASfqde00bQbjU7410nRcXjm6DFqQyH6SylSboGAx4PtuJQcF59pkckX6TjYh9Vr4wlY2LbLcy8d4czpYRrvOYOrHExcRDyIeCi6eUYLGdKZNLV+wWMWSRcL7Orr4tb6ZhoDCb5Qt5mwdyNnuof53tM5cvlHEIGRSJ7SCptV1RcSe9jjoyc3xqn0MGtjDfP425qeouNwZGSAvkyK2kCI9dX1+C39J61p70S/Oq4DpRSHBwfoSSdpikRwLRflKiiVrzR1UdiuC65irDdFa3UcB4XXY9HaUMXprgGyZ/pYtbyenFlF0QERhSEmI7k0FhaWuBTdKH7LgyB0jA2zpc7AY0TJ5Uv820/24PNZ1NeWB2gHCz2c7BymPV6DddGyDILgXDIIvBCligX+af9O+jJpTKMcc7U/xB9uuoMq/+Vr/GiapmfpzLmS4/CtA/v4xv49nBgf4c3+bjo6ejG7iuTOFCicyOMZVagi4CosBXnXJlMqsqaqnIwt02BsoHxFatTbSEHqQWWAArZbwsXFUSFMPIDCZwpFpxeftQyftYyTZ4colBxCwQtz55f5qnAch+Hx9Pm2omNjILSFFv7FT8+f7aAvk6YlEqUxFKElHGO8kOOnp47Od2iatmDpHv4c29Hdyb7+XtpicbLFEp1nhijuTROIuDhBg7whGOMuq2JRCvUu5qBLzBegPRKnOlBZvkCZNCbqSdujRDwJqow7ONm5DzF6GU0GSJdWYjdtoiG0G585gO24uGoVTdHPIWJQLNqXbUUWwU9LvoqhQho3p0CBEni8dSthz8KfF79noIe64MT1/2uDId4a7OXTa7dg6OmjmnYZnfDn2I6eLqoD5fXi19TUkHxzmFGPoDAJuUKDz4vlCupklv/8W+/mpDNA/2CSKl+gvJtUMkfA7+GxzY+wN/tzRnL9dJzsY7wvxvi+lST7AvSTp6t9nDvf91sUSTGcK/I7G7djGeXlkluaqspr1Dvu+VU1HUexLJfgw+s3M6DSmGKwLt5ArX/qBdkWAsswK6WnC9NWXaUwDXPSfTY1TdMJf865Sp3f6dc0DOLKQ7g6ThaHrcFq3j7RTbFYgixkXx/iQ+/dyvMvH+HtE/0ANNbHeezdG6mNRnko8inePLyXg/v3MHrIwcJLXdTC53jp7Uix99cnWP/Acn57/VY2114YdK1NhLnr1uW8tuskHssAEWzb4f47bmJDQzMb5uMXc43e1dTKj08epTUSQ0RQStGXyXBfc7u+OEzT3oFO+HPs9sZmnuk4StDyICKEG8J0HxmgOhTg5L5OggEvQcuDG1bsf/UYXsvk8cdvJ5sr4rqKUNB7PoGZYuKMREid8oOTwxuyEIS4FcD0Qe2Inz/ZeAfhyMRBSxHhgTvXsKq9jmMdfRRsm4Dfiy/goW8oSX115B2TZF/3KDtfPkZf9yj1TXFuv2cNTcvmv8Z/T/NyulJJ9g/1YYjgKpc1VTU82r56vkPTtAVLJ/w5dmdLK8dGhnh7ZBgRwW724D/lQfoLKMB2FY7tsG5FLfGIn/2vHeeux7YQDF6+zZ/tupzydtEZPgGmIu/zEi4k8JZCCELQ7yWbKV6W8KGc9FubEijgqWd3U7RtQEAp7tjczrvvXHtZ0u85O8xTX3sZy2MSCvvpPDXIiaO9fPz37mVZe83c/MKmyWuafGb9VnozKYbzWeK+AC3hqO7da9oVTGdP268D7wcGlFIbK20J4HtAO+U9bT+ulBqV8qvty8B7gSzwOaXUnrkJfXHwWxZ/sGUbJ8dG6U2nqPIHqL87wJP/9Rn6iw6xaICm+sj5BcSUUuSzBfyXJHzHdXnywK/YX3qZ0WaTkVEvph8SuUEauvPUxesIBL1E4++8Tr1tOzz93F78PovqeHnA03Vddrx1mpva6mhvnniF7Su/OozXZxGrbI7u9YVJjmd55bmDfPoP7p/F39LVERGawlGawrO3taOm3cimMy3zn4H3XNL2F8DzSqnVwPOVnwEeA1ZXvp4AvjI7YS5upmGwOlHNva3t3FxXT111lHfdexN1XqG1NkQkVJ4uWcgV8QV9RCZJ2keGBziaeZmupEO+QWH5XdwC9IVMetpKxGIB7r5vLT6fRd/pQd58bj9vvXKE1MiFaZe9Q0my+RKhwIXpmYZh4LVMjpzsv+wxezqHiUQnflqIRAP0do6gLtoYRdO0xWE6m5i/LCLtlzR/ELi/8v03gReBP6+0f0uVs8EOEYmLSKNSqne2Al6ojo0M8lLnacbyOdZW13JPS/s7XgB06PVj7PrZXjoPnuHknpPE62MsW78Mx1G87zP3YE6yLvwbgy+RdkdJFmoI+13UqgKFIYv8mI9sXLjzjnVs27qSX//bDva8cAgxBKXAsgw+8IWHWbFxWbncMUmeVkphGpeXQhI1YXLZIqGLli/OZYvEr1Dz1zRt4braC6/qzyXxyr91lfZmoPOi47oqbTe0N3q7+Kd9b9KVGqfkurzSdZr/c8dLvNp1mjPJ0Qm94b5TAzz79V8Tr3G55cEwbet8FFJpRruG+fgXH2H9bZfvZ1t08+Slg0KhXFoxRDC9JoGmEpG1eWra/ERbonR39LP7+UPUtiSoW1ZNfWs1oViQZ7/xIsVCiYaaKJGQj1Qmf/7c55Z2WLfi8qUUtt+3luRolnyuvEFKPldkfDTDnfevme1foaZp18FsD9pO1u2b9LO/iDxBuexDa+vi3ce06Dg803GUumAIv2XhuIrBbIZT46N0psZpjkRYHkvwuY23EPJ4OfSb44icRRkDGIZB0wpF4woY74sQjl7oSTvuOAX7LAZe8kRJBAIEzSIKhYtbWQJBYRou9YF64r4Ab+97G4/PxLhoqQR/yMfQ4Di/eHM/0bYEd9+9ml+/eJTeoSSgEIT7b1tFS8Pla+LftKGZ933sNl751WEGescJhn089pFtrNu07Hr8ajVNm2VXm/D7z5VqRKQRGKi0dwEXZ4MWoGeyEyilngSeBNi2bduiLQiPFfIUHJvqQLl805kaoy+TJu7zg4KmUJRTYyP87ORxPrpmI6mxs2D0IRJHkPJEGUo47klGhoc4dGaQY6d34w8eYsPaAjUJByVBvOLlrsY6upKjZIslfFYJQwziZhNrq5pZEUvQ4zG5dBmcoWyG/UN9dHV24HciKOBdd7aywVtDyXZpro+TiE0+0CsibNjaxrrNyygWbLw+C8PQq3Fo2mJ1ta/eZ4DPVr7/LPCji9o/I2XbgfEbvX4f9ngQylMmATpT4wQtD45ShLzluff1oQhv9HXhuC5tG/LkMxNr6cWciYPiqz94kW8+9Twdp/Zy+FiQH/60noMn4uw5201pZD+nRk7y0DIft9TWEjETxK1qHm29hf+04TYMEW7a2o5jO9il8o5Ztuuy+/RZAhE/K1Y20RyK0hgI81rfGbzVXm6+qekdk/3FDMPAH/DqZK9pi9x0pmV+l/IAbY2IdAF/Dfwt8H0R+X3gLPCxyuHPUp6S2UF5Wubn5yDm685xhlDuCIZRjWFOnLoY9Hi5q7mVlzpP0xgO4yqF4ypcpVgWKZdJRMoDowpYuaWeZWstuo4V8QcNbFtRKrrQBv2DWaoTGXAL5HOjOAEvz7xS4O4HLaq9fmpzbfSlBlmVKPD4TZtZF9tOta/2fCwN7bXc/7HtvPTvb4BSJAsFXEfR9oF19L5wirFDAyAgq6Psq+5ixc3zfwGVpmnXz3Rm6XzqHW56aJJjFfBH1xrUQqFUkVz2+xQLuxAMFC5e3x0Egh9F5MI68u9bsQYD4WfHjpEaL5BxC9zW0ELUV55LP5TNcnNtPZZhEApv5b1/8EtOHwpzcKeDjUGibZzfHKsiP+IyXhoh2FCg4C/hljIUS2GGChmOnw1x9pQf213HUNTPpz75EIWxEmOBLPHohV76toc2smZrOz2nBukvZkgmTzDw7FnyQ1m8iQAoGN0/yIHiPj60YeM79trHxrP0DSTx+z20NFVNWEJZ07TFSV9pewWF/AsUCzsxzGWIGCjlUiy8jmHU4g88fP44yzCQMyXqT5jU+eo5xDBHTvcxlslRUx3Gb1nc3dIGgGk2EQh9gmP9P+Ws8uGJJRk2BhmmHhLdZHIunqCLKgiuDW6pxOG9taTzfixDIcDpI/382ZeeZmVTDSjFmlUNvPfhm/H7ym9CkUSYNYkwK1yHnzx1jKG+JNHGGACucrFq/MhAnu4TA/9/e3ceJMd1H3j++97Lq+6u6q6+0QAaN0DwBE+RuiXLuqiRLEsa2+MJa0OxsePdmY2NWNt7xe7OP54/dmfWGw6N5V3POjwztix7PJZliaIkiiJtkSLBEzcI4uhu9N11H3m//aOKECCCFkk0CKI7PxGI6szOyn6/V4Vfvcp8B1t2Xdk7R2vNk0+9wo+febW3DRQLKX75M3dTKl45O2Uikbi5JM22N6C1xveeQKpRhOhVkxASqYbxvSeuOHZ+qc5Lx+eYKBfZMlDgfcMF7i07BKsVGm4LX/l89fhT/LuTh2kHPidPj3F+9hATOyYY3Fen66ZxVJcgNjGHAsJQopXGDRxyw3VadYXjGASiw0qlRVD1qbse6bRFeSjPqTOLfO9Hx18XgykVHyptJdaapu/R8F3aYcC+wWEypkV9rfW655ybWeWJp19haDDLyHCe0eE8na7PNx95MRlslUjc5JIW/s9wwy5HVk5ztlJjl5yluWgQRHXsnGR0ZJzywAhCd694zux8tTf/uhFQKT1HYNYx4oii1SSfGWM0dx9og+PVRf7yLPgvdygW8qjB4zQaJhJFymyQUzWaF1P4kaKhHPYdmCOf7dC4kCUt12h50FoVOIai2nU5tbTKoe0TlIdyHDs5z4ce2kc6deWUDHu2jnFbeQyjnCLWmgHbwVaK5VqFgfLrp0I+cvwiace8NI0ywEAhzcJSnUqtzWAxe9V601oTxb3OWkoOJwOzEol3oU2b8JtNl9m5NWqey5Ld4WJrjtbas1Tt87gIlBK0nHnGh5roIEcUCF5dWsPVrzA19IUrzpVyDLTWNHNnCMwGbtdmvtPA7SrU4DLV8DQl4wBj6TxHKotsDVO9DwirTuiZdJoGr740hhsaSOFjz3eY2LZGaqLDs5VJ1rIFlIoZyXfJ+g1cN4WVsllptWl5PlnHRgN+EL4u4W/ZPcbE9jIL51cZHC2gI83SxQpTe0YZ317mZ4VhjPiZUbdCiF6//+jqLfwwnKfR/hPCqNchS6kRCplfwzAm3/4L9HM0ah1eePYsF15dpjSU5c57d67rLJ5aa7woxJAKI+mdlNggNmXCP3J0lu989wi1rsurnQWG9rzC2LZzmKM+qutgdi0iV7LgZBg3a0jdJkMaoQMqDbCLoxQuO9/ObcNYtkHTnMHrGKx5LUI/QCpNhxjXO858M0+gY6JIcP+uMU4+s4DTLeGKDi+8UCAyBNIKcXWKTstGHWtyrlxiYnuTrkzTbZrMNFMU2i2oQkCHQMfUJruIGAq5FPns66dyUEry2f/ywzzz6BGO/uQMCMG9v3CQez5y8Ko3bPfvGeXkKwsUcqlLrfRW2yWfTzFUen3rXmuPWvOraB2iZG9QdRxXqTW/Smngf0KK9V9ftlHr8O//8HG6HY9sPsXZ00ucPHKRz3zpXnbuHb/m85+vVfnm6ZPM1OvYhuKhLdv40PZpTPX6KS8SiZvJpkv4tVqHb3/3CAOFFCeDVaYOncPJzpPJdGhFNobSWFmXZiOFLwTPNqdIBTHbYpPYHWBuzeGu0QUmXZ8ojsmmbDJpmy988hD/7uSTrDZboCOUERCXNIgIPw6Yb58lY4zgRyHHS3OMTAxw+KksNdMh8jW2FaC9kGABYt8gsDIYf9VlYTpD5sNtvPMp8BXtnYrsMUmEpLPcZnm2ijWh+MLDdyGvMh8OQCrr8L7P3s37Pnv3z62fXTtGObBnnGOnF3rz62gwLcUXPnHnVc/vB6eJ4waG8dPxdkqUCKMZ/OAUjnX723+x3sBzT5+h2/YZHut1e02nbTptj8ceOcL07tFrGi+w3G7xb597BtswmMjlCOKY7559hW4Y8I/27l+vEN512oHPhUYNQ0q2F4qYMvlwu968qEonmEdJh6w5hbys59/1sukS/rnzK+hYE0nQdhUn28LRPlrL3gCqSGIZIcrorQMbIFnuFoncEnbLIrYifvxMxOPVH/ZG0g4V+PT9B5iaKPE+9R7+5rWwqYgAACAASURBVMKjyNhFSIOlBUGgQmp2Di/ySMuAe0YmCWVEdZuPbI5ROx2TVstkHJe5So5wwEBLQWxJZJQie6FG8HSImQ9wcj6xrbBNh24QIaWB3fR56NAOTCl54cWzlEZccoUaQphkrT2Y6sqpg4M44my9QjsImMzmGU5f2Wo3lOTjH72V8S1FlpYaTIwU2L1jlGzG5mq07oC42qUegY476/Oi/YzzZ5bJD1z5zSGdsVlerNNpe1ddD+DNevriHMClie8spZjM5XlqbpaPTO8ka71+nYKb3bOLc/zFmaO9JSM15Cyb3zhwiC25ws9/cuIt01qz0H6c+c7jCEBrsFSeXQO/RsoY/rnPvxabLuG/1tHEkAJpBmgNUmpkpDHsGILe76UShIFEmhqhBaYP3a7LxfYQWVlg62AOAaw12vzxo8/ym595kLsGH+TbF39Ee6ZD5cc2Wmq6XYuumWLkQZf7txqk7SxeFPD9p85QmfXQkcuY4zHfKuCbBsoLEEpCoIkdSWc4R/Z0FXVXRJxSePOSoOMhQnAaLg1V4fe/9j1Wmm123zHPgYOvUipk2b9nAtOwmcz9Kjl7LwAr3TZ/eOQZKt5Pbzo/OL6VT0/vv7To9/nFCn/xw5do9ydMO1utMT5RfMOEb6gt/XqNL/Vm0joG9BWt/vVUKGa4OLuGc9n9ijCMUEpi2dfWSlpsNUmZV55D9b8xNH1vwyX8pU6Tr59+maFUGlv10kHdc/mjY4f5H+55f9LSvw6awTkutn9AyhhFil6de1GVs/VvsL/0X13XDg+b7m7Utm1DvZuQsaAoy0SxphvkMHSMEWmydhchNZEWdAOHer3AoPQRZkBuuogZTbJ9cAIpBEIIirk0HS/g1MwyOTPL7uBuZl/ME97iER/0se51yY+1GXk5Imf2ksWZl5aZXXDp5H3EsM9KKUc4oBBRDFKgARWGoGNC20QbinbWplZJEb5iIiKw673yXpA+M34blfNxGxFr3RznLgiOnQgwRI755p8Rax+tNX966iXaYcBENs9ENs9oJsvjc+c4vtabC7/V8fiPjz6HEDA6mGN0MIcXBPz7R5/D70/X8LOUGiNlv5comiGK1oiiNcJoBse+H0Ndn5u2hx7YSbfj47q9D6UwjFhZbHDnvdNY1rW1YaYHirR8/4p9fhShpKDoOG/wrJvXkdUlEOJSsgco2A7NwONCo3YDS7ZxVdyXUcK5lOwBLDmAGy7TjZb/gWdeu03Xwi8VM3z4g/v5/mPHGIiKrF0cJR6dpR3nSMctLAmxjiiYglT+Ng4OP8i0vR/D0lxc1CyfP8FMVGMhbmCimFJFhIRqu0sYxbx8ZIXUdBe/axDHEkSMNebROG9w+tkKbupVLq61KY6kkbkIrTXxMQjX7N4ne6QReYFWoIIIHQvaKk2rmSX3TBXD1TihQPgxwWieVsrAFoKBfJfQM2i2bUrpgIuLVe68ZRuRqNENZnHjMWaaNcYzP+2KqYQkZ1k8uzTHLUOjvDK3gh9EDBZ+OsCqkE2xsNbgwmKVXVte36tHCEE2/Vkscw+u/yxojW0dwrYOXreWytT2Mp/6/N388JEjNGo1pJTc+9AuHnj/vms+9z0Tk/x4bpaFVpOSk8KLQqquy8N79uIY1/8a6zvNj6Ort/r0T+eHSqyvWIeXvg2/RojeRIro6Lr+7U2X8AHuunMb09vLXJhdQ4jb8ItzrHCcOKqQjU12DdzKQPZWTFXAkj+9lBEONDgmFvGDkJQwidAsRHW2WjEyZfPHf7fArH4VoyOIQiDWgMC7qFCyy6o+TVyzMIOQiXKLlmlT/2tBZqWDTuUIizlohei2AkegpUIYMcGURbplIm8bQoUu/oIAbRAULAg0QvQWO0GAAjwF0ouJ4ri3QwjiNxg0JURvmmWArnf1VjwavDdo4ffOIbGtg9jWwbf1erwd+w5uYfe+cVpNFydlYTvrk4zztsNv3n0vPzx/luMrKxScFA/v3setI69fL2Aj2Fss8/0LZ4i1vnRZzw1DDCnZmn/9lNmJa1dybmHNfRFLDlxK/EHcwpQ5UsbIdf3bmyLhd0OPI6sLtDoBW9JFVipNOkFI2bCpzbdQKyXu2vd5RiZ7b3DXDzg3V8H1VxgvFxguZhFCsBLMc+/WM5RZpBUWeKU5TiZ7GtOu8sj8k3hRzOCEwcrpIloILn2Iz4F5R0TomhAZRBrEWky61MX1DeKSgaU9YtsiGnOIO0DWAEcT7/JRNYvM2QA3IxElSaasaZ8xQYPQmiiOadaz5MYapPMuQSTIZtIo1QWRIWVsIW0YjKSz1Dz30g1J3R+Be+dwryvjVH9O/DjWl3rkhFEMQjBZfvf951eGurTe7noqpdJ8bt8tfO7avzC8623PF3lochtPzp1H9S8nCuBLe24jtQG/0bwbFKxdDDl3sOa+CEKC1khhsWvg1xDi+t4z2dAJP9Yxfzv7Iv/33/+EypLfW93Jg5QwSQcK3w25xcizzyzw9BOn+NDHb2V0eoj/8J3DtF0frXtv/ntv2cpH7hnFcL/KB8ZXqbcV9e5ZRkovcLJV5IX2MNpXgMbO+Fhbu3ivZqD3WmIQEZmCbkshRYA2JbGWqFaMMaRxV03CQCHdNlJ4RL4kuMXBmAoJVyxENoZIYFZiusLEz4ExFOGsWoAg8iLakcLP2zjpFgUtOLirCCJkS/6fXuru9aU9t/G1I89wsdkAoYk13FEe59ahMQAmhgoc2ruFZ47P4NgGsdb4QcSHDu1i4Bp6viTevYQQfGZ6P7cPjXG6uoqpFLcMjryu91Zi/Qih2Jb/LEOpQ7SCCxgizYC9B1O9fuT7uv/td8P8KIcOHdKHDx9e9/M+u3qG//nx71GdDXEcSaXpE8YxIpKkXUVBmHRFxCeHtjEmU1TWmojtBSJiCtkUXhxy0l3ldHONz25/hd0DJzgRODQESCXRTofTzUHW2ilsT/Q+qe0Q1zQIVixUS4AdI17QOLf6+A0LqTXKMIiiCNtos/psimYjhyV8pONCDGJN0/mlAkHWwqpDuqFJv+LQ7QooSMjBxw4tELTfw1MvVRAKzLxByw/YNqT4Z5/awfbJMdLmNEpeeaOxE/icqCzTDHymcgNsyxcvfZWHXuv+7Pwax84tYijBgekxto4Uk6kSEol3ISHEc1rrQ2/2+A3dwn90/hjNSohtSsI4Jop7NyojpfFUhIGNjGOOd6psKWbpBBH1pSq7d4wSxBGPd8/T8FoMhWeQznFORFCzJKJrIHSMGyq6mDhGhO5IkCACiAJFo5nCNsDzJR0MJis1jEJI3JKIMMJyYrxUnposEJckkZFChimcxSp6RKFtA+WbBOmI7BmYTCsqoYcVQUlGOMsRrxyrkO1qSvk0ZZnjk796iDt2T17qRng1adPirpE37j0jpWDn5BA7J4eux0uSSCRuoA2d8BuBC5FAKImOLutxIED3GuRILQguWxdQSsli1OTv3HMstKocnDjPIE1CrcHSOMInSkdErk2sX5slXwAatMAwY9o1m/icyYrIEKQk4aDg3GqKtOtRzDfJWR2aUZrm3DhGqYGeD4gFhJaJN5SHhzTagriridcscoGPEmAJhWW6BJ2QU8cG8aRJsWxx545J2g2XY0+e5a49yXqziUTi6jZ0wr+1NMFPsiv4lQjTlggBMf2BVLEgjmNCGbPNydFqdCnlUzCieGTlOJ3FFsVMg+J4k2bX5pQeYnd+FRlplAiJPUUmjnglBi3BsCOkhJZr49UspIyRVoyMJdLRRBlJt5qm00hjuiEhJnauSXo0ws3aGFGvq5ZvFMgELeSKpF03ELaBrxXNto8UMJgHgcVafZztYyUmBwuYSlIYSLG83GBttcXQVWbBTCQSiQ2d8D8ydpBndlzg6cYqQVdgKnB9jaElecuh1Q0oa5uBukBkBZ/70r38yekf0T3XQAqFkw0xzBjZ0DSkRaWSIlvyUCLGSvksz+eQdUV32KTrGbieTegqzCZEzYB4KEYOeNiBQFVMhB0RCUlgKkw3Qvkm2gwQNkShgTA0ItDEniJIS2LHYHsjQ2R5ZDMjHNie570P7eTMcY8L51YYuKyHymszWsZR0nc6kUhc3YZO+MNOgf/10MM8Uj7K90+/ituCXfkyqchCo7l9yzg70nlSlsXwSIeFxr/k3GybAbNAKh3jRQohez2niGF1KY0vBHYqpDafoVJL42rF3JlBIlOws1Th/i0XGB1r4O42eezCHmbW0kyPtamm8wgrBkeDGTGWr3BuZoyOa2Blu+iugdu1SOVcOmmJZzsMD0g++YEdrAV1HirfwoPlAwAYYp4Tx+cpDOhLN1PbLY9M1mbwKq37WMfEOkYJldx8TSQ2sWtK+EKI80ATiIBQa31ICFECvg5sA84Dv6y1rl5bMd++opXlS7vu40u77rtif6w1zyzP8J8WTtCtNbm/+ycsuavU2rupmxa2aOH4mnbXJF10aTVtjE4MNUmnZuMZiotrQ8zkC8SB5I6BBT49eoSVbobFuIDT8Xho8Cw/rm3FvajwERRGXVwMyukG28prZAdczi2VWayXUEQMiDqjW1ZYUqMEaYu7h0d6a8kGYFzWP3fn7lEOHJzgxNGLSCnRWmOais998b4rFi6JdcyLtSO8XDuCH/uU7SEeGLqXEef6Du5IJBLvTuvRwv+A1nr1su3fBn6gtf5dIcRv97d/ax3+zrp6dO4kj86dZtDOUIxO8J1vK049dRtBU6InBOd2DZGJuiy9mmHLeJUtZoXs9oDQM4i04sTf76Dt5VBOhAo07504y6qbxsPEbZiceWy4N+eNhqWlHOFITHvEREnNRKYKnoQu7N47hzcjMKKA4W4dbMWBoRWUtZVdhTJB3BvhujM3dqnsSkk+8fBd3HbHNuZm1nDSFrt2j5LLX9lX/tnKczxffYmSNUDOyNIImvzN/CN8bvJhita7byBVIpG4vq7HJZ2Hgff3f/5j4HHeZQm/Hfg8Pn+WiXSBlj/H818/x9IP80S1BiKC0pE6mVN51j5aJHeyzeqLOdLvCUnlW2g3zXNntyIqGnuqS802MIOYnOOx0MoSa8HcE0VErDHzEUWjQ6Vu4lctSs0aw1MddCAJY4Vd9lB2yOBoHeY1hYLmvq0lzmhBLFusunWEgF8cPcSQfeVUtVIKprYNMbXt6t0nvcjjSP0YQ3YJ1f92kDOzVPwax+oneLB8/3Wv50Qi8e5yrQlfA48KITTwB1rrrwEjWusFAK31ghDiqhM8CyG+AnwFYGpq6hqL8eZ1ozZPLD/JufYJqqHEnauw/ERM3A3AUmAIRKhxztUpvirJ6w4f+/RZssMR0tKksprthRkaOww6OYOnWtPU2ynm3RwFx2VxIUfYkaQLAZYR0WwaFHUb3/AodZqUh1s4ToST9TjSGianNaIJ2x3Be3aXOTQ+xtZzLV5dMxga28vde3ZQTL31UY/dqNubdfNnhmo70qbiV9apNhOJxM3kWhP+e7TW8/2k/j0hxMk3+8T+h8PXoDfS9hrL8aYEsc/jS/+ZFb+GIUz8qEF9pkFYCyGnoCt6PeoNARKsix32f2qFrOXSVik6HZPu8w7zLw+w0kjRNSQ8EGNsiXissZVf2fIyedNEKU3KDBBhzMxCCVuH2EGM1DA8XMfH5MzaMGkFeWI+dYvFdG4KFdn82TdiKlWBaeaZOTrH3EsNvvj5e0inrz4f/RvJGBmUUARxiCl/+jJ3Y5ddzs51rtlEInEzuKaEr7We7z8uCyH+CrgHWBJCjPVb92PA9Z3g+S1Y6F6gEdYYTZXZmjvFkbU2iAhiCEMDEWkU/RkmlSCKBNsHVlkhQ2clzZlHt9A6b2PaEUiJDhTxd1zCA4qFAzn+1LuNuwtzGFZEdc5goZWn6xt4UiHbYIQtTreH0UIzku8wYBe5Y+gWdltLKOXy3R9Are4zMmxhWtsQwmZxqc5TT5/hQx888JZiNaXJoeId/P3q0+SMDKa0aIYtLGGyP7/3+lRwIpF4V3vbCV8IkQGk1rrZ//mjwP8OfBP4deB3+49/vR4FXQ9Vb5W5ExGHjy+y1nRxxqCGBX4XIwwJUgZIQSoTgBUixwVPNfbT9FK0FrJYkU8x28YwNd3QIvIcRC4kc8HDnciwODvIt0UBsxyQvhAiqxFKaWKt8EYiXprexrGTDu/dY/HpHQ+yJbebtCoRBkfwuo9z6pUWQ4OTmNZ2hOi16AdLGY4cu/iWEz7AwcIB0irNS7UjtKM2O7PbuX3gNnJmMjFWIrEZXUsLfwT4q36/bgP4j1rrR4QQzwJ/LoT4MjADfP7ai7k+Th92ef4Jl8paG3fJQ33Xw/QV1h0K92iA44dkp1wINMaEpvCZGMfpUKh0OefbVNNpYi0Y7TQJdW8qWSKTbj6NWRcEviTCIVIWwX1dSn4bIwqJ0hJj2OSOoXHGtuZ4cHQve4s/ne/ItG7FMA9i2Y+gjPQVU6TGMVd0tXwrhBDszE2zMzd9jTWXSCQ2gred8LXWZ4HbrrJ/DfjQtRTqeuh0PL733AKn2x4ajZxMoSfTOCd9smsR4nabwXIF2QU5aCD3WXhCkAlbpPIhWkjS0qeVdui2unjaBKXxTIvANjClj235xJ4ijgRWHGEXOjipEv6OiHQhx87SEA+U9/JQef/ryieE4Pbbpjj83HlGhvP9RU00lUqLhx7cfQNqLJFIbDQbeqTt5WaWqxxtLGNVOwhHI0JN5AnaU2msYy5mJqb0YejUMohYENoK/BhfG5CL8CxB7Bv4gUGtk8KQMSKAIK8Q5ZhSvkOoJH5gEnQEYcvEewxEscnA4QxyzGLiV8f4wMhBtNa8sDbHE4tnaIYe+wujvH9sFw8+sJuVlSYXZtYuJfxdu0a45+6khZ5IJK7dpkn4ldAlDLooCaEwCT2JJAZTE6ZNZCdGoxA6JsDEVRLDk4Smibfm0Fky0cpAGhppC7SriIYDjB2wp7xKSvi9m78pyVqYYu5cEVXVyH1pyOcx1zR//5fP849vuY+/Wz7Lt+eOU7RSONLg8NoMJ+qL/Nf73scXf/le5hdqNBpdigMZRkbyyXQIiURiXWyahJ9J2eh0xJBuk84EVLppVltpIkuAEeEaOSqzHRpli9Ugiwg0WkvGnSr+rEGUFwg/YmCtTfkDVdR4yHljgPvjs8yeGEc7GseK0L4mZUeMpVcoT9XJl0LOuwb50jC1sw1Wqg1+MH+KsVQeU/au1Y+m8sx36jy3OsMHxnczMV5kYrx4g2sskUhsNJsm4dNc4wv3HmZU1cDTCANOLJX5/sldmBmXbrbEqeUJjJxHzuz2+uMLzfnKEGYjIhd2MXWIo0Lqr6SpjFtklMstk3MErsPC/BCx5yNiKOUaHLh9gdZ8EVM1GDZNOtEAOdOh7XvEWl9K9q/JGBbn28mAqEQicf1s+ITfjVz+9syPmFv8A6bGVlnp5ggihdAR2VLMlu1tKnYRtRrRGbCxFgSRYSDQBKFJrGIsJ6I42yaOwLdNui0THUGhGWKMaaanFhgfW6UQhxTTbWyjy8wTg1SEIooiSmad+UrEHbt3MTo0APMQ6Rglftr7phv6jKbyN7CmEonERrdhEn7La7FQnyPnFBjN9yYa01rze0f/gifPPM+vbl1ltZvBjAOMfMCPj+zFDwxsI6ZVyyDTEZYKSA/4RK6i27UglNjC692MNW0MK2SYJu6kYml1gGXytAOHnO1iGhEyjrGzHoOqy3IQES9HaENgSIfbBid5+FfeSyvw2ZopcaK+xJbMAKZU1PwuSkruHnrnpphIJBKbz02f8OM45psnvsH3z/2I1TVFKhVx69Qo/8Xt/y3HKos8PnMUfVZgpiNYEbi2w0xjkM6azUCxTWxJlBUzMNRCRGnsoosjQwLfoDmfpi1s7IamGSl0KHBFivSWKmFTYgjN0+d28r7pExTNDkJDMe6QM+DuT66x9UCHxpzNmj7I8I5p/sUjX2c+bpPOKXJDNg2vTTmdZ1t2kE9N3cKQkwyISiQS189Nn/CfOP8j/vAbz9E9OtybcljFzA57VD7/fxBaBwl/EiNPK87nhxga7tCom1TrGQwzwgw1F1czGG5IuKKwsjFxoKhKk0AbhGMCGpBe8lGxwCwEiILGVJLdu+dZXBzgxPwYS+08A/kWU/U2xtQsA9uWwfCZ3O3R2OHwey9nac98mzAlyGifThuWqimmttb5yp6Pc//oHTe6GhOJxCZw0yf8P3z0UbqHM0QpiMoxwgdjweYn/9klOngUYyZmcEeTo5UxPjV9hFy6w2y7iN8p0uykWFooEGYFddtC+gKn4xOkFbEAW4V0TAO9O6agur3r+r6iMZfHHAgYG6/SjBzmvTwr5/NUl33u2TtHO3ZIEXDcLfI3a7too4jyYHQiRjNN7p2cwSLiqdWdfOvcd7lv5DaEeHujaROJROLNuukT/uKzBq0xQWu7Qvdzpl2NGDhloYcb7N69zAPvOQORYnElxdzLOWbPOJxzBvEMk7BkgoqRgQbLpx3aZAouhcglDhXdIEVNpRnQHloLpNDoSKKXDaJMTDls0z2exxYBzSGDb1R2MqYb7HEWeaExTiXIgIQYwcHBJX6xfAovMsgIn89ln2c5XqUTLJCxJm5sRSYSiQ3vpk/4HSVpbDfQujc9AZGmWzIQW2NKTsxnPvw8thnz+LE9HPlWGde16Gobaj4qLdGGJMoZxCpGGxqrERFoRYjE9WyIJVppPBQpHeG7BrbjIlyNRqDakJlzkfsCwlHNWrPAkG7zwplddCspaqKE2KpxSgGD6TbLbhaNINISD4MDQ/Po+CKQJPxEInF93fTXERpbJLEhMFWME4TYIgIh6IwLPnH3CxQyHY6sjnHx5TRGPURLRRwqzDjCbrYozSwznl+iVGxiql7/fCE0wtSkMi7CiIhjQbdj4bdNlBWRtjtoCWYmonnBwk65yFdicMGpR6w+OUJzOYs2NHHTxH7epFD1mbAaIOhNwSw1CMVIxsDQnRtdjYlEYhO46Vv4Ih1juSGhI3AzBjLS2K0QHM13/u0+Hhu+nUYrQ9iWoDTWxRYyFSEymlQcE7csxFFFJhWSHveIdvssBjmC2KDgdMmWWzTbKWQqoJCvkHVcRFsiFDTPWnTPWKQzAaYXw7KJKSVxAKlcl2qURqdgPNVm21KDDz5wnjBWLHXSLLo5pgcEo+ltSJW70dWYSCQ2gZs+4dudmOqQiRYCtCZSkmBQ4bR96qUy1BTajcAx0LbEyztkT1cwTYh8uzdzZhYiLVEzgun9F1mNM4S+QS3MULC7EGu0jFk4nWdQCbIrHu1KirVaEa0FnZZBSkYMFluoGUU64xMJQTvKMZ3y2JttU1kbxJYWKcOjZLc4YI0ymN6LlBLD2HejqzGRSGwCN3XC972ATsYiRqDaAhkIUBCmYry0yY70MtpKURc5YimAmMA1ae8skb3QQYoYWRQgBLGQZAotOG1z363nCRwTEcYM5ho8eWE3ONAdNKlXc4SVSVTJpZwNaVdBdiwGRocYnGjTbWmCVpa0HGInJiX5MmldplBOMV3+DEHwEkR1EHWkzJLO/jpSpm90VSYSiU3gpk343Y7Hn//RkwSGwqxJpA/Qm1XS7ArCQkzgg9vOEmgTGUPsgMpHxEIQDNgIM8TMgPA1A6pFLurgL1qkxgMy21wypsfc0iCUYuIy4EHYKXDrx/ZROB6i4xhVkExuH2bwrjGeOHOB0W2S04fnCIXEtARxGLBWlXz2EwIpc9j2e4jjFjpeIJ3/LaQ0b2Q1JhKJTeSmTfgvPv0qK4s1jIwgDgXKjVCeJlaCIC2RHcVsZpi0qUiFPqFrIFxBqEycchdfKqaH5tk7uYgpQtbms6xdKOB5BtZFSWlnl6af5tWwjJ6M0Z7CNAUHRif5l7/yWQIvZG2pju1YlIZznJhbRp+5wEA5y647J5g9vUK74WJmczz8oTr796b6JReAj2HemiT7RCLxjropE77WmieePMpLu5/DfGEA5bmkF33QgNYEBYP2qEO55LJ3aBkRac6tlFlaLKJcjS8ko7evMFqoYgY+jgzZessK2bzLU3++Ay+WXCiXWKmWMA60MVwDQoUyBf/NL3wUKQS2YzK+dehSmXaODVLKplmutxgqZ8kNpVmutdgxWuae238M8QJaZIE2oDBTn7pR1ZdIJDap65bwhRAfA/4vQAH/j9b6d9fr3MeOX+T58hHc8TbW4xkyhVfZMWET1xUyFVPPCy4+PcCezyySyoUYluYD245xfq7M0+d30s0YuGmD5ztTqDYc8ObJ45ErthgcabDQGqLSymOEkDYV0rCxHIP3j+9n3+j4VctkGQa/8cFDPPLCKY7NLSER3Lljkl+4fTcp835C7ymi6AJK3Y5hP4BUw+tVHYlEIvGmXJeEL3qrcP8+8BFgDnhWCPFNrfXxaz13HGse+7ujhDu65KRHeXSO0qk8zZaFNGLiSKCEZsfdVc797Raa90jE9phsymWqWGGw1qRlGGSdLlGkCGzFyWCcu/QF4kAwOlXlrFUitbWNM2sylIOxwih7CuP8kx3/8FK9A5kUX3zwdoIoQiAwLi0+PoiV/uS1hp5IJBLX5Hq18O8BzvQXOkcI8WfAw8A1J/wwjKh2OkhT4K21GG/kaTRN7IIP/aUAw65EnLPxtr1E/titVIcNDEOz7OdwMh7aCZCRJhYaK4poYdHWFkUVsTSeQw+aZCqSXSN5xvxpvnzwIcac0pteatBU6ucflEgkEu+w6zXSdgKYvWx7jnWaO8A0FUOpHKaX4uSjLu6ShZGKLiV7AMOJ8FoG+dFJhCsxGho3MknJgBopcuk2gW9gmRG6K9ACjJJPZ8rkYrFMrhahGlNkjd3sH9jBeGowWVc2kUjc9K5Xwr9adtRXHCDEV4QQh4UQh1dWVt78iYXg/e/dz9iFKQIvwrB7l3Gu+ENxrwSujPpTGIAhI8JI0okNQqVYXCvQXHLQJmQHXFQq4tyLE0zOuoxYFrlCCkeZ3L9vP/F4HQAABj9JREFU69uJP5FIJN51rtclnTlgy2Xbk8D85Qdorb8GfA3g0KFDV3wY/Dz7943zFfUJMhJmZ39C9BMHZUVIQ4AGv2WSm+oQX4B2PoKiIOu4nG0MsXtwjrOzo3RWHDoNG0tFlITPuXAbzkDI+O4VTi4d5IHpSX7p/oOMl5JlBxOJxMYgtH5LufbNnVQIAzgNfAi4CDwL/GOt9bGrHX/o0CF9+PDht/W37vutX+aWoED9ZBYd9xJ+dqzL4k7QzxWZP5QiLgoMI2QitUrqTMTqMwW0hvHpNYp3dcG12D7YZrjssGfqy9w2dR8py0wu4yQSiXc1IcRzWutDb/b469LC11qHQojfBL5Lr1vmH71Rsr9WT/+rP2f6wBgHf+MzFLwI15I8ffIsW+p7WRvzCOomhU6VHTYYjRTzZprtn2+xdauFq/ezb3gfn9o5hYEkl5pGCut6FDORSCRuuOvWD19r/W3g29fr/Jc7e2yh/zcjQL6uZd5uuSzMVTEMycTWQaqBR7XboZRKU05n3okiJhKJxA13U460fSO97v+vl8k67Nw7dml72DQYThJ9IpHYZG76BVASiUQi8eYkCT+RSCQ2iSThJxKJxCaRJPxEIpHYJJKEn0gkEpvEdRl49ZYLIcQKcGGdTjcErK7TuW4mSdybz2aNfbPGDa+PfavWuvxmn/yuSPjrSQhx+K2MPNsokrg3n80a+2aNG6499uSSTiKRSGwSScJPJBKJTWIjJvyv3egC3CBJ3JvPZo19s8YN1xj7hruGn0gkEomr24gt/EQikUhcxYZJ+EKIjwkhTgkhzgghfvtGl2e9CSH+SAixLIQ4etm+khDie0KIV/qPxf5+IYT4vX5dvCyEuPPGlfzaCCG2CCF+KIQ4IYQ4JoT45/39Gzp2IYQjhHhGCPFSP+7/rb9/uxDiJ/24vy5Ebz5vIYTd3z7T//22G1n+ayWEUEKIF4QQ3+pvb5a4zwshjgghXhRCHO7vW7f3+oZI+KI3TebvA78I7Ae+JITYf2NLte7+P+BjP7Pvt4EfaK13AT/ob0OvHnb1/30F+Oo7VMbrIQT+O631PuA+4J/1X9uNHrsHfFBrfRtwO/AxIcR9wL8C/nU/7irw5f7xXwaqWuudwL/uH3cz++fAicu2N0vcAB/QWt9+WffL9Xuva61v+n/A/cB3L9v+HeB3bnS5rkOc24Cjl22fAsb6P48Bp/o//wHwpasdd7P/A/4a+Mhmih1IA88D99IbdGP0919639NbbOj+/s9G/zhxo8v+NuOd7Ce2DwLfordG9oaPux/DeWDoZ/at23t9Q7TwgQlg9rLtuf6+jW5Ea70A0H8c7u/fkPXR/7p+B/ATNkHs/csaLwLLwPeAV4Ga1jrsH3J5bJfi7v++Dgy+syVeN/8G+O+BuL89yOaIG0ADjwohnhNCfKW/b93e6xtlAZSrLT67mbsfbbj6EEJkgb8E/oXWuvEPrDe8YWLXvSXcbhdCDAB/Bey72mH9xw0RtxDik8Cy1vo5IcT7X9t9lUM3VNyXeY/Wel4IMQx8Twhx8h849i3HvlFa+HPAlsu2J4H5G1SWd9KSEGIMoP+43N+/oepDCGHSS/b/QWv9n/q7N0XsAFrrGvA4vXsYA0KI1xpql8d2Ke7+7wtA5Z0t6bp4D/BpIcR54M/oXdb5N2z8uAHQWs/3H5fpfcjfwzq+1zdKwn8W2NW/k28BXwS+eYPL9E74JvDr/Z9/nd717df2/5P+Xfz7gPprXwlvNqLXlP9/gRNa6//zsl9t6NiFEOV+yx4hRAr4ML2bmD8Efql/2M/G/Vp9/BLwmO5f2L2ZaK1/R2s9qbXeRu//8WNa619hg8cNIITICCFyr/0MfBQ4ynq+12/0TYp1vNnxceA0veuc/+ONLs91iO9PgQUgoPfJ/mV61yp/ALzSfyz1jxX0ei29ChwBDt3o8l9D3A/S+5r6MvBi/9/HN3rswK3AC/24jwL/S3//NPAMcAb4BmD39zv97TP930/f6BjWoQ7eD3xrs8Tdj/Gl/r9jr+Wx9XyvJyNtE4lEYpPYKJd0EolEIvFzJAk/kUgkNokk4ScSicQmkST8RCKR2CSShJ9IJBKbRJLwE4lEYpNIEn4ikUhsEknCTyQSiU3i/wcCpA+WGwlUMwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x295017b83c8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# YOUR CODE GOES HERE\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "colors = np.random.rand(X_train_level2.shape[0])\n",
    "plt.scatter(X_train_level2[:,0], X_train_level2[:,1], c=colors, alpha=0.5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 0.000000; Corresponding r2 score on train: 0.409477\n"
     ]
    }
   ],
   "source": [
    "alphas_to_try = np.linspace(0, 1, 1001)\n",
    "r2_train = np.zeros([np.shape(alphas_to_try)[0],])\n",
    "# YOUR CODE GOES HERE\n",
    "index=0\n",
    "for alpha in alphas_to_try:\n",
    "    r2_train[index] = r2_score(y_train_level2.clip(0,20),(X_train_level2[:,0].clip(0,20)*alpha + X_train_level2[:,1].clip(0,20)*(1-alpha)))\n",
    "    index += 1\n",
    "best_alpha = alphas_to_try[np.argmax(r2_train)] # YOUR CODE GOES HERE\n",
    "r2_train_simple_mix = r2_score(y_train_level2.clip(0,20),(X_train_level2[:,0].clip(0,20)*best_alpha + X_train_level2[:,1].clip(0,20)*(1-best_alpha))) # YOUR CODE GOES HERE\n",
    "\n",
    "print('Best alpha: %f; Corresponding r2 score on train: %f' % (best_alpha, r2_train_simple_mix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best alpha: 0.000000; Corresponding rmse score on train: 0.820599\n"
     ]
    }
   ],
   "source": [
    "alphas_to_try = np.linspace(0, 1, 1001)\n",
    "rmse_train = np.zeros([np.shape(alphas_to_try)[0],])\n",
    "# YOUR CODE GOES HERE\n",
    "index=0\n",
    "for alpha in alphas_to_try:\n",
    "    rmse_train[index] = np.sqrt(np.sum((y_train_level2.clip(0,20)-(X_train_level2[:,0].clip(0,20)*alpha + X_train_level2[:,1].clip(0,20)*(1-alpha)))**2)/y_train_level2.shape[0])\n",
    "    index += 1\n",
    "best_alpha = alphas_to_try[np.argmin(rmse_train)] # YOUR CODE GOES HERE\n",
    "rmse_train_simple_mix = np.sqrt(np.sum((y_train_level2.clip(0,20)-(X_train_level2[:,0].clip(0,20)*best_alpha + X_train_level2[:,1].clip(0,20)*(1-best_alpha)))**2)/y_train_level2.shape[0]) # YOUR CODE GOES HERE\n",
    "\n",
    "print('Best alpha: %f; Corresponding rmse score on train: %f' % (best_alpha, rmse_train_simple_mix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.82059863, 0.82059887, 0.82059944, ..., 0.96866464, 0.96893872,\n",
       "       0.96921299])"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test R-squared for simple mix is 0.352241\n",
      "Test R-squared for single LGB is 0.352241\n"
     ]
    }
   ],
   "source": [
    "test_preds = pred_lr.clip(0,20)*best_alpha+pred_lgb.clip(0,20)*(1-best_alpha)# YOUR CODE GOES HERE\n",
    "r2_test_simple_mix = r2_score(y_test.clip(0,20),test_preds.clip(0,20)) # YOUR CODE GOES HERE\n",
    "\n",
    "print('Test R-squared for simple mix is %f' % r2_test_simple_mix)\n",
    "print('Test R-squared for single LGB is %f' % r2_score(y_test.clip(0,20), pred_lgb.clip(0,20)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9144167633544937"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sqrt(np.sum((y_test.clip(0,20)-test_preds.clip(0,20))**2)/y_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.sqrt(np.sum((y_train_level2.clip(0,20)-(X_train_level2[:,0].clip(0,20)*alpha + X_train_level2[:,1].clip(0,20)*(1-alpha)))**2)/y_train_level2.shape[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# YOUR CODE GOES HERE\n",
    "lr = LinearRegression()\n",
    "lr.fit(X_train_level2, y_train_level2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train R-squared for stacking is 0.276578\n",
      "Test  R-squared for stacking is 0.349031\n"
     ]
    }
   ],
   "source": [
    "train_preds = lr.predict(X_train_level2) # YOUR CODE GOES HERE\n",
    "r2_train_stacking = r2_score(y_train_level2,train_preds)# YOUR CODE GOES HERE\n",
    "\n",
    "X_test_level2 = np.c_[pred_lr, pred_lgb] \n",
    "test_preds = lr.predict(X_test_level2)# YOUR CODE GOES HERE\n",
    "r2_test_stacking = r2_score(y_test,test_preds)# YOUR CODE GOES HERE\n",
    "\n",
    "print('Train R-squared for stacking is %f' % r2_train_stacking)\n",
    "print('Test  R-squared for stacking is %f' % r2_test_stacking)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stacking: whole model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dates_train = dates\n",
    "\n",
    "\n",
    "X_train = all_data.drop(to_drop_cols, axis=1)\n",
    "X_test =  sub_test_data\n",
    "\n",
    "y_train = all_data.loc[:, 'target'].values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = LinearRegression()\n",
    "lr.fit(X_train.values, y_train)\n",
    "sub_pred_lr = lr.predict(sub_test_data.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lgb_params = {\n",
    "               'feature_fraction': 0.45,\n",
    "               'metric': 'rmse',\n",
    "               'nthread':11, \n",
    "               'min_data_in_leaf': 2**4, \n",
    "               'bagging_fraction': 0.84, \n",
    "               'learning_rate': 0.122, \n",
    "               'objective': 'mse', \n",
    "               'bagging_seed': 2**7, \n",
    "               'num_leaves': 2**4,\n",
    "               'bagging_freq':1,\n",
    "               'verbose':0 \n",
    "              }\n",
    "\n",
    "model = lgb.train(lgb_params, lgb.Dataset(X_train, label=y_train), 100)\n",
    "sub_pred_lgb = model.predict(sub_test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dates_train_level2 = dates_train[dates_train.isin([27, 28, 29, 30, 31, 32, 33])]\n",
    "\n",
    "# That is how we get target for the 2nd level dataset\n",
    "y_train_level2 = y_train[dates_train.isin([27, 28, 29, 30, 31, 32, 33])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27\n",
      "28\n",
      "29\n",
      "30\n",
      "31\n",
      "32\n",
      "33\n"
     ]
    }
   ],
   "source": [
    "# And here we create 2nd level feeature matrix, init it with zeros first\n",
    "X_train_level2 = np.zeros([y_train_level2.shape[0], 2])\n",
    "\n",
    "# Now fill `X_train_level2` with metafeatures\n",
    "for cur_block_num in [27, 28, 29, 30, 31, 32, 33]:\n",
    "    \n",
    "    print(cur_block_num)\n",
    "    \n",
    "    '''\n",
    "        1. Split `X_train` into parts\n",
    "           Remember, that corresponding dates are stored in `dates_train` \n",
    "        2. Fit linear regression \n",
    "        3. Fit LightGBM and put predictions          \n",
    "        4. Store predictions from 2. and 3. in the right place of `X_train_level2`. \n",
    "           You can use `dates_train_level2` for it\n",
    "           Make sure the order of the meta-features is the same as in `X_test_level2`\n",
    "    '''  \n",
    "    M=15\n",
    "    X_train_m = X_train[dates_train.isin(list(range(27-M,cur_block_num)))]\n",
    "    y_train_m = y_train[dates_train.isin(list(range(27-M,cur_block_num)))]\n",
    "    X_test_m = X_train[dates_train==cur_block_num]\n",
    "    lr = LinearRegression()\n",
    "    lr.fit(X_train_m.values, y_train_m)\n",
    "    pred_lr_train = lr.predict(X_test_m.values)\n",
    "    \n",
    "    lgb_params = {\n",
    "               'feature_fraction': 0.45,\n",
    "               'metric': 'rmse',\n",
    "               'nthread':11, \n",
    "               'min_data_in_leaf': 2**4, \n",
    "               'bagging_fraction': 0.84, \n",
    "               'learning_rate': 0.122, \n",
    "               'objective': 'mse', \n",
    "               'bagging_seed': 2**7, \n",
    "               'num_leaves': 2**4,\n",
    "               'bagging_freq':1,\n",
    "               'verbose':0 \n",
    "              }\n",
    "\n",
    "    model = lgb.train(lgb_params, lgb.Dataset(X_train_m, label=y_train_m), 100)\n",
    "    pred_lgb_train = model.predict(X_test_m)\n",
    "    \n",
    "    index1 = dates_train.isin(list(range(27,cur_block_num+1))).sum()- (dates_train==cur_block_num).sum()\n",
    "    index2 = dates_train.isin(list(range(27,cur_block_num+1))).sum()\n",
    "    X_train_level2[index1:index2,0] = pred_lr_train\n",
    "    X_train_level2[index1:index2,1] = pred_lgb_train\n",
    "    #  YOUR CODE GOES HERE\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# YOUR CODE GOES HERE\n",
    "lr = LinearRegression()\n",
    "lr.fit(X_train_level2, y_train_level2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_test_level2 = np.c_[sub_pred_lr, sub_pred_lgb] \n",
    "sub_pred_stack = lr.predict(X_test_level2)# YOUR CODE GOES HERE\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Submission LGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_test_data['item_cnt_month'] = sub_pred_lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>5037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>5320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>5233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>5268</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  shop_id  item_id\n",
       "0   0        5     5037\n",
       "1   1        5     5320\n",
       "2   2        5     5233\n",
       "3   3        5     5232\n",
       "4   4        5     5268"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[['ID','shop_id','item_id']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>shop_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>target_lag_1</th>\n",
       "      <th>target_item_lag_1</th>\n",
       "      <th>target_shop_lag_1</th>\n",
       "      <th>target_lag_2</th>\n",
       "      <th>target_item_lag_2</th>\n",
       "      <th>target_shop_lag_2</th>\n",
       "      <th>target_lag_3</th>\n",
       "      <th>target_item_lag_3</th>\n",
       "      <th>...</th>\n",
       "      <th>target_item_lag_4</th>\n",
       "      <th>target_shop_lag_4</th>\n",
       "      <th>target_lag_5</th>\n",
       "      <th>target_item_lag_5</th>\n",
       "      <th>target_shop_lag_5</th>\n",
       "      <th>target_lag_12</th>\n",
       "      <th>target_item_lag_12</th>\n",
       "      <th>target_shop_lag_12</th>\n",
       "      <th>item_category_id</th>\n",
       "      <th>item_cnt_month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6425094</th>\n",
       "      <td>5</td>\n",
       "      <td>5037</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>1052.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>1092.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>119.0</td>\n",
       "      <td>...</td>\n",
       "      <td>54.0</td>\n",
       "      <td>991.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>105.0</td>\n",
       "      <td>954.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>1445.0</td>\n",
       "      <td>19</td>\n",
       "      <td>0.383015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6425095</th>\n",
       "      <td>5</td>\n",
       "      <td>5320</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>55</td>\n",
       "      <td>0.238589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6425096</th>\n",
       "      <td>5</td>\n",
       "      <td>5233</td>\n",
       "      <td>1.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>1052.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>1092.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>150.0</td>\n",
       "      <td>...</td>\n",
       "      <td>37.0</td>\n",
       "      <td>991.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>119.0</td>\n",
       "      <td>954.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19</td>\n",
       "      <td>0.866186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6425097</th>\n",
       "      <td>5</td>\n",
       "      <td>5232</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>1052.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>1092.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>23</td>\n",
       "      <td>0.224132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6425098</th>\n",
       "      <td>5</td>\n",
       "      <td>5268</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20</td>\n",
       "      <td>2.459580</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         shop_id  item_id  target_lag_1  target_item_lag_1  target_shop_lag_1  \\\n",
       "6425094        5     5037           0.0               25.0             1052.0   \n",
       "6425095        5     5320           0.0                0.0                0.0   \n",
       "6425096        5     5233           1.0               42.0             1052.0   \n",
       "6425097        5     5232           0.0               28.0             1052.0   \n",
       "6425098        5     5268           0.0                0.0                0.0   \n",
       "\n",
       "         target_lag_2  target_item_lag_2  target_shop_lag_2  target_lag_3  \\\n",
       "6425094           1.0              110.0             1092.0           3.0   \n",
       "6425095           0.0                0.0                0.0           0.0   \n",
       "6425096           3.0               80.0             1092.0           1.0   \n",
       "6425097           0.0               48.0             1092.0           1.0   \n",
       "6425098           0.0                0.0                0.0           0.0   \n",
       "\n",
       "         target_item_lag_3       ...        target_item_lag_4  \\\n",
       "6425094              119.0       ...                     54.0   \n",
       "6425095                0.0       ...                      0.0   \n",
       "6425096              150.0       ...                     37.0   \n",
       "6425097               65.0       ...                      0.0   \n",
       "6425098                0.0       ...                      0.0   \n",
       "\n",
       "         target_shop_lag_4  target_lag_5  target_item_lag_5  \\\n",
       "6425094              991.0           1.0              105.0   \n",
       "6425095                0.0           0.0                0.0   \n",
       "6425096              991.0           2.0              119.0   \n",
       "6425097                0.0           0.0                0.0   \n",
       "6425098                0.0           0.0                0.0   \n",
       "\n",
       "         target_shop_lag_5  target_lag_12  target_item_lag_12  \\\n",
       "6425094              954.0            1.0                65.0   \n",
       "6425095                0.0            0.0                 0.0   \n",
       "6425096              954.0            0.0                 0.0   \n",
       "6425097                0.0            0.0                 0.0   \n",
       "6425098                0.0            0.0                 0.0   \n",
       "\n",
       "         target_shop_lag_12  item_category_id  item_cnt_month  \n",
       "6425094              1445.0                19        0.383015  \n",
       "6425095                 0.0                55        0.238589  \n",
       "6425096                 0.0                19        0.866186  \n",
       "6425097                 0.0                23        0.224132  \n",
       "6425098                 0.0                20        2.459580  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub_test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_file = sub_test_data.merge(test[['ID','shop_id','item_id']], on=['shop_id','item_id'], how='left')[['ID','item_cnt_month']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_file[\"item_cnt_month\"] = sub_file.item_cnt_month.fillna(0).clip(0, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_file.to_csv('sample_submission1.csv',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Submission: Stacking "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub_test_data['item_cnt_month'] = sub_pred_stack\n",
    "sub_file = sub_test_data.merge(test[['ID','shop_id','item_id']], on=['shop_id','item_id'], how='left')[['ID','item_cnt_month']]\n",
    "sub_file[\"item_cnt_month\"] = sub_file.item_cnt_month.fillna(0).clip(0, 20)\n",
    "sub_file.to_csv('sample_submission1.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.15"
  },
  "widgets": {
   "state": {
    "eca1b7547a504f5981d90fa3f8d0f31d": {
     "views": [
      {
       "cell_index": 10
      }
     ]
    }
   },
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}