Computer_Vision/MNIST_LeNet5_Ensemble_TensorFlow_High_Accuracy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "685eb9f6",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "papermill": {
     "duration": 0.028458,
     "end_time": "2022-04-19T13:16:55.987401",
     "exception": false,
     "start_time": "2022-04-19T13:16:55.958943",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Implementing LeNet5 on MNIST Kaggle Dataset - Ensemble with TensorFlow - Accuracy=99.75%\n",
    "\n",
    "### [Dataset Link](https://www.kaggle.com/competitions/digit-recognizer)\n",
    "\n",
    "\n",
    "# [Link to my Youtube Video Explaining this whole Notebook](https://www.youtube.com/watch?v=yWLK8YSYhxo&list=PLxqBkZuBynVRyOJs4RWmB_fKlOVe5S8CR&index=18)\n",
    "\n",
    "[![Imgur](https://imgur.com/PfxgcSh.png)](https://www.youtube.com/watch?v=yWLK8YSYhxo&list=PLxqBkZuBynVRyOJs4RWmB_fKlOVe5S8CR&index=18)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "42a8df38",
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_kg_hide-input": true,
    "_kg_hide-output": true,
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
    "execution": {
     "iopub.execute_input": "2022-04-19T13:16:56.053601Z",
     "iopub.status.busy": "2022-04-19T13:16:56.052657Z",
     "iopub.status.idle": "2022-04-19T13:17:01.494012Z",
     "shell.execute_reply": "2022-04-19T13:17:01.493411Z",
     "shell.execute_reply.started": "2022-04-19T13:08:42.290240Z"
    },
    "papermill": {
     "duration": 5.479132,
     "end_time": "2022-04-19T13:17:01.494181",
     "exception": false,
     "start_time": "2022-04-19T13:16:56.015049",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D, BatchNormalization\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccaa8179",
   "metadata": {
    "_uuid": "cd31c62c12088bfa6f2b26dcecc714182627c767",
    "papermill": {
     "duration": 0.027239,
     "end_time": "2022-04-19T13:17:01.549413",
     "exception": false,
     "start_time": "2022-04-19T13:17:01.522174",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Load 42,000 training images and 28,000 test images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "662bcb2c",
   "metadata": {
    "_uuid": "d71b3fa2b10620dc8870352fc18d4548f824a88a",
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:01.610381Z",
     "iopub.status.busy": "2022-04-19T13:17:01.609866Z",
     "iopub.status.idle": "2022-04-19T13:17:06.455804Z",
     "shell.execute_reply": "2022-04-19T13:17:06.456256Z",
     "shell.execute_reply.started": "2022-04-19T13:08:47.582725Z"
    },
    "papermill": {
     "duration": 4.878227,
     "end_time": "2022-04-19T13:17:06.456420",
     "exception": false,
     "start_time": "2022-04-19T13:17:01.578193",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"../input/digit-recognizer/train.csv\")\n",
    "test = pd.read_csv(\"../input/digit-recognizer/test.csv\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "92fe0277",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:06.524859Z",
     "iopub.status.busy": "2022-04-19T13:17:06.523279Z",
     "iopub.status.idle": "2022-04-19T13:17:06.539880Z",
     "shell.execute_reply": "2022-04-19T13:17:06.540286Z",
     "shell.execute_reply.started": "2022-04-19T13:08:52.943105Z"
    },
    "papermill": {
     "duration": 0.054791,
     "end_time": "2022-04-19T13:17:06.540420",
     "exception": false,
     "start_time": "2022-04-19T13:17:06.485629",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>pixel0</th>\n",
       "      <th>pixel1</th>\n",
       "      <th>pixel2</th>\n",
       "      <th>pixel3</th>\n",
       "      <th>pixel4</th>\n",
       "      <th>pixel5</th>\n",
       "      <th>pixel6</th>\n",
       "      <th>pixel7</th>\n",
       "      <th>pixel8</th>\n",
       "      <th>...</th>\n",
       "      <th>pixel774</th>\n",
       "      <th>pixel775</th>\n",
       "      <th>pixel776</th>\n",
       "      <th>pixel777</th>\n",
       "      <th>pixel778</th>\n",
       "      <th>pixel779</th>\n",
       "      <th>pixel780</th>\n",
       "      <th>pixel781</th>\n",
       "      <th>pixel782</th>\n",
       "      <th>pixel783</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 785 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   label  pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  \\\n",
       "0      1       0       0       0       0       0       0       0       0   \n",
       "1      0       0       0       0       0       0       0       0       0   \n",
       "2      1       0       0       0       0       0       0       0       0   \n",
       "3      4       0       0       0       0       0       0       0       0   \n",
       "4      0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "   pixel8  ...  pixel774  pixel775  pixel776  pixel777  pixel778  pixel779  \\\n",
       "0       0  ...         0         0         0         0         0         0   \n",
       "1       0  ...         0         0         0         0         0         0   \n",
       "2       0  ...         0         0         0         0         0         0   \n",
       "3       0  ...         0         0         0         0         0         0   \n",
       "4       0  ...         0         0         0         0         0         0   \n",
       "\n",
       "   pixel780  pixel781  pixel782  pixel783  \n",
       "0         0         0         0         0  \n",
       "1         0         0         0         0  \n",
       "2         0         0         0         0  \n",
       "3         0         0         0         0  \n",
       "4         0         0         0         0  \n",
       "\n",
       "[5 rows x 785 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0b071176",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:06.613842Z",
     "iopub.status.busy": "2022-04-19T13:17:06.613121Z",
     "iopub.status.idle": "2022-04-19T13:17:06.616094Z",
     "shell.execute_reply": "2022-04-19T13:17:06.616545Z",
     "shell.execute_reply.started": "2022-04-19T13:08:52.972041Z"
    },
    "papermill": {
     "duration": 0.046508,
     "end_time": "2022-04-19T13:17:06.616678",
     "exception": false,
     "start_time": "2022-04-19T13:17:06.570170",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pixel0</th>\n",
       "      <th>pixel1</th>\n",
       "      <th>pixel2</th>\n",
       "      <th>pixel3</th>\n",
       "      <th>pixel4</th>\n",
       "      <th>pixel5</th>\n",
       "      <th>pixel6</th>\n",
       "      <th>pixel7</th>\n",
       "      <th>pixel8</th>\n",
       "      <th>pixel9</th>\n",
       "      <th>...</th>\n",
       "      <th>pixel774</th>\n",
       "      <th>pixel775</th>\n",
       "      <th>pixel776</th>\n",
       "      <th>pixel777</th>\n",
       "      <th>pixel778</th>\n",
       "      <th>pixel779</th>\n",
       "      <th>pixel780</th>\n",
       "      <th>pixel781</th>\n",
       "      <th>pixel782</th>\n",
       "      <th>pixel783</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 784 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  pixel8  \\\n",
       "0       0       0       0       0       0       0       0       0       0   \n",
       "1       0       0       0       0       0       0       0       0       0   \n",
       "2       0       0       0       0       0       0       0       0       0   \n",
       "3       0       0       0       0       0       0       0       0       0   \n",
       "4       0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "   pixel9  ...  pixel774  pixel775  pixel776  pixel777  pixel778  pixel779  \\\n",
       "0       0  ...         0         0         0         0         0         0   \n",
       "1       0  ...         0         0         0         0         0         0   \n",
       "2       0  ...         0         0         0         0         0         0   \n",
       "3       0  ...         0         0         0         0         0         0   \n",
       "4       0  ...         0         0         0         0         0         0   \n",
       "\n",
       "   pixel780  pixel781  pixel782  pixel783  \n",
       "0         0         0         0         0  \n",
       "1         0         0         0         0  \n",
       "2         0         0         0         0  \n",
       "3         0         0         0         0  \n",
       "4         0         0         0         0  \n",
       "\n",
       "[5 rows x 784 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a183fedc",
   "metadata": {
    "_kg_hide-input": true,
    "_uuid": "b3c56055d1ba56d28d982f9647c33439c46753ff",
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:06.762026Z",
     "iopub.status.busy": "2022-04-19T13:17:06.760962Z",
     "iopub.status.idle": "2022-04-19T13:17:06.922979Z",
     "shell.execute_reply": "2022-04-19T13:17:06.921929Z",
     "shell.execute_reply.started": "2022-04-19T13:08:52.989245Z"
    },
    "papermill": {
     "duration": 0.277016,
     "end_time": "2022-04-19T13:17:06.923128",
     "exception": false,
     "start_time": "2022-04-19T13:17:06.646112",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "Y_train = train[\"label\"]\n",
    "X_train = train.drop(labels = [\"label\"],axis = 1)\n",
    "\n",
    "# Given images are not already normalized (i.e. the pixel values are in the range [0,255]).\n",
    "# So normalize them by dividing by 255\n",
    "X_train = X_train / 255.0\n",
    "test = test / 255.0\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3b76ac80",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:06.987276Z",
     "iopub.status.busy": "2022-04-19T13:17:06.986170Z",
     "iopub.status.idle": "2022-04-19T13:17:06.989229Z",
     "shell.execute_reply": "2022-04-19T13:17:06.989593Z",
     "shell.execute_reply.started": "2022-04-19T13:08:53.239069Z"
    },
    "papermill": {
     "duration": 0.037292,
     "end_time": "2022-04-19T13:17:06.989722",
     "exception": false,
     "start_time": "2022-04-19T13:17:06.952430",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42000, 784)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.values.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff959c87",
   "metadata": {
    "papermill": {
     "duration": 0.029227,
     "end_time": "2022-04-19T13:17:07.048647",
     "exception": false,
     "start_time": "2022-04-19T13:17:07.019420",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Now note, the axis order for an image tensor in Tensorflow is as follows:\n",
    "shape=(N, H, W, C)\n",
    "\n",
    "* N — batch size (number of images per batch)\n",
    "* H — height of the image\n",
    "* W — width of the image\n",
    "* C — number of channels (usually uses 3 channels for RGB)\n",
    "\n",
    "\n",
    "On the other hand, the shape for image tensor in Pytorch is slightly different from Tensorflow tensor. It is based on the following torch.Size instead:\n",
    "torch.Size([N, C, H, W])\n",
    "\n",
    "* N — batch size (number of images per batch)\n",
    "* C — number of channels (usually uses 3 channels for RGB)\n",
    "* H — height of the image\n",
    "* W — width of the image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "65794145",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:07.112647Z",
     "iopub.status.busy": "2022-04-19T13:17:07.111885Z",
     "iopub.status.idle": "2022-04-19T13:17:07.114542Z",
     "shell.execute_reply": "2022-04-19T13:17:07.114997Z",
     "shell.execute_reply.started": "2022-04-19T13:08:53.246550Z"
    },
    "papermill": {
     "duration": 0.037025,
     "end_time": "2022-04-19T13:17:07.115123",
     "exception": false,
     "start_time": "2022-04-19T13:17:07.078098",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train = X_train.values.reshape(-1,28,28,1)\n",
    "X_test = test.values.reshape(-1,28,28,1)\n",
    "Y_train = to_categorical(Y_train, num_classes = 10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0998452",
   "metadata": {
    "papermill": {
     "duration": 0.02933,
     "end_time": "2022-04-19T13:17:07.173959",
     "exception": false,
     "start_time": "2022-04-19T13:17:07.144629",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## to_categorical\n",
    "\n",
    "Converts a class vector (integers) to binary class matrix. A binary matrix representation of the input.\n",
    "\n",
    "## Why we need to_categorical\n",
    "\n",
    "A classification model with multiple classes doesn't work well if you don't have classes distributed in a binary matrix.\n",
    "\n",
    "The reason you want to_categorical (even on numeric labels) is due to how the relationship between your labels is understood by the algorithm.\n",
    "\n",
    "For example, suppose you made a color classifier. You mark red as 1, blue as 2, and orange as 3.\n",
    "\n",
    "Now you feed them into the machine learning algorithm to help decide what your input matches. The math is going to say that orange is higher than red. This obviously isn't your intent, but the network will know that orange is greater than red.\n",
    "\n",
    "### Another example\n",
    "\n",
    "Suppose you have three clasess, the vector goes like this:\n",
    "\n",
    "- [1, 0, 0] = class 1     \n",
    "- [0, 1, 0] = class 2    \n",
    "- [0, 0, 1] = class 3\n",
    "\n",
    "You use `to_categorical` to transform your training data before you pass it to your model. \n",
    "\n",
    "If your training data uses classes as numbers, `to_categorical` will transform those numbers in proper vectors for using with models. You can't simply train a classification model without that. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "446b2bb8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:07.237149Z",
     "iopub.status.busy": "2022-04-19T13:17:07.236520Z",
     "iopub.status.idle": "2022-04-19T13:17:07.239109Z",
     "shell.execute_reply": "2022-04-19T13:17:07.239496Z",
     "shell.execute_reply.started": "2022-04-19T13:08:53.255878Z"
    },
    "papermill": {
     "duration": 0.036015,
     "end_time": "2022-04-19T13:17:07.239610",
     "exception": false,
     "start_time": "2022-04-19T13:17:07.203595",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42000, 28, 28, 1)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3886411e",
   "metadata": {
    "papermill": {
     "duration": 0.028563,
     "end_time": "2022-04-19T13:17:07.297531",
     "exception": false,
     "start_time": "2022-04-19T13:17:07.268968",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Augmentation \n",
    "\n",
    "Following Augmentation have been applied on this set of 42,000 images training images.\n",
    "\n",
    "### 1. rotation_range:\n",
    "\n",
    "To randomly rotate images through any degree between 0 and 360 by providing an integer value in the rotation_range argument.\n",
    "\n",
    "When the image is rotated, some pixels will move outside the image and leave an empty area that needs to be filled in. You can fill this in different ways like a constant value or nearest pixel values, etc. This is specified in the fill_mode argument and the default value is “nearest” which simply replaces the empty area with the nearest pixel values.\n",
    "\n",
    "\n",
    "### 2. zoom_range = 0.10,  \n",
    "\n",
    "\n",
    "### 3. Random Shifts with width_shift_range and height_shift_range\n",
    "\n",
    "It may happen that the object may not always be in the center of the image. To overcome this problem we can shift the pixels of the image either horizontally or vertically; this is done by adding a certain constant value to all the pixels.\n",
    "\n",
    "ImageDataGenerator class has the argument height_shift_range for a vertical shift of image and width_shift_range for a horizontal shift of image. If the value is a float number, that would indicate the percentage of width or height of the image to shift. Otherwise, if it is an integer value then simply the width or height are shifted by those many pixel values.\n",
    "\n",
    "\n",
    "\n",
    "### 4. vertical_flip and horizontal_flip\n",
    "\n",
    "ImageDataGenerator class has parameters horizontal_flip and vertical_flip  for flipping along the vertical or the horizontal axis. However, this technique should be according to the object in the image. For example, vertical flipping of a car would not be a sensible thing compared to doing it for a symmetrical object like football or something else. \n",
    "\n",
    "\n",
    "### 5. brightness_range\n",
    "\n",
    "Brightness can be controlled in the ImageDataGenrator class through the brightness_range argument. It accepts a list of two float values and picks a brightness shift value from that range. Values less than 1.0 darkens the image, whereas values above 1.0 brighten the image."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0abf887",
   "metadata": {
    "papermill": {
     "duration": 0.029574,
     "end_time": "2022-04-19T13:17:07.359146",
     "exception": false,
     "start_time": "2022-04-19T13:17:07.329572",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Plot Images before Augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0095e793",
   "metadata": {
    "_kg_hide-input": true,
    "_uuid": "b95ca2c1e71cb5457684eff3c35bb8d68b4a0f97",
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:07.428973Z",
     "iopub.status.busy": "2022-04-19T13:17:07.428186Z",
     "iopub.status.idle": "2022-04-19T13:17:08.702925Z",
     "shell.execute_reply": "2022-04-19T13:17:08.702277Z",
     "shell.execute_reply.started": "2022-04-19T13:08:53.265839Z"
    },
    "papermill": {
     "duration": 1.314213,
     "end_time": "2022-04-19T13:17:08.703093",
     "exception": false,
     "start_time": "2022-04-19T13:17:07.388880",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAEDCAYAAAAC+VQaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABH6ElEQVR4nO3decAN9f///xHZskaFypJClkpokewqhJAlWZJ3iiTR4h3Zk61SRKVUlqIQ3qESSsnaXpSUSIRkl73fH59fzx4z33OuOee6zn7db389jjNz5nWdzpk50zzn+cry999/OwAAAACA4M6I9wAAAAAAINFx4gQAAAAAPjhxAgAAAAAfnDgBAAAAgA9OnAAAAADABydOAAAAAOAjm8/z9CoHAAAAkJlkCfSPXHECAAAAAB+cOAEAAACAD06cAAAAAMAHJ04AAAAA4IMTJwAAAADwwYkTAAAAAPjgxAkAAAAAfHDiBAAAAAA+OHECAAAAAB+cOAEAAACAD06cAAAAAMAHJ04AAAAA4CNbvAeQqOrVq2d56dKlll977TXLHTt2jOmY0uvPP/90PT506JDl5557LuA6q1evtty9e3fL+fLls3zjjTe61smSJUuGxpnsTp06Zfmhhx6ynDVrVssjRowI+O9AOP7++2/Lv//+u+u5CRMmWN6+fbvlyZMn+75u586dLQ8aNMj13AUXXGD5jDMy3/9zC/b9/uSTTyyvXbvWcs2aNS1797MVK1aMxhABIMNOnDhhWX8LvvPOO0HXOXz4sOXx48cHXOaaa66x3LZtW8sdOnSwnCtXLtc63seJIPMd/QAAAAAgTJw4AQAAAICPLFryEUCaT6aaOnXqWF6xYoXlkydPWtZSPb28mAgOHjxoedGiRZbbt2/vWk4vw4aiTJkylrdu3Wr5jjvucC33yCOPWC5ZsmRY20gFf/31l+XcuXP7LpMzZ86ojymWSpcubbl8+fKWZ8+ebTl79uxRH4e+x47jOB988IHlJk2aRH370XL06FHLuh+65557YrL9J5980nKvXr0sp3LZnu4rO3XqZPmNN96wfPPNN1suUKCA5TfffNPymWee6XrdWbNmWb7pppsiMlYAkbFhwwbL48aNs3zs2DHLu3btspxWCdtVV11luUWLFpYbNmxo+bLLLkv/YCNkx44dlgcPHmz5xRdfjOk4hgwZ4nrcv3//mG7fI+D9J6l7xAMAAACACOHECQAAAAB8ZPpSvWHDhgXMx48ft9ymTRvLL7/8suVg5VixtG/fPstaOpjWpeNoKVKkiOV58+ZZLlu2rOX8+fPHdEyxlNlL9bZt22b5kksusayd3QoWLBj1cfz222+ux82bN7e8Zs2aqG8/krRTUfXq1S1//fXX8RiO0fKVHj16xHEk0fXf//7XsnbE7Natm2XtYqjq1q1redmyZa7n8uTJY/nbb7+1XKJEifQPFill9+7dlvX75jjuTo7ez9Y/tDy0cePGlsuVK2dZj81et9xyi2X9vGbLljrNmPX2hkcffdTylClTAi6j9LdzejoK6/G/devWll999dWwXysSdF/3+uuvW96zZ4/lI0eOWK5ataprfS3ZPvvsswPmdevWWd64cWPAcVSoUMH1uEaNGpYnTpwY/A+IDkr1AAAAACA9OHECAAAAAB+ZrlRv7ty5rse33XabZe2Yol1OPv74Y8t58+aN3uDS4d1337WsXVoSiV5ejVUHsHgIpVRPJ8LUiYVTjU6UrKWukyZNivq2vaV6OnHrhx9+aLlWrVpRH0tGbdmyxXIidarUEp8+ffpYvvPOOy0n4wTPc+bMcT1u166dZf2bteTE2zHvH9rNVLucOo57UvIxY8ZY1vcy0eh78/7771vWUtjChQsHXb948eKW//jjD8ta/hOK5cuXux7rMf3SSy+1rKVXuu1EoOXLWlav3RYXL14cdP0cOXJYLlq0aMBlTp8+bVm74abHFVdcYVm7S2qZbjKU8On+1HHcx4Bg71GjRo0sa1fYjJbqffHFF5Z1EvOuXbtaHj16tGudWHSldRz3e6ElytoBVMs/HSe0/b1+75966qmA2/DS7+4vv/ziu40Io1QPAAAAANKDEycAAAAA8JEpSvV+/fVXy02bNnU999VXX1kuVKiQ5cmTJ1tOtEkztXRwwIABlrUMKT2effZZy8WKFbOspSSrVq0K+3W1I4++r61atQr7tRJZKKV6N954o2Uts0w1OjmyfsdWr15tOVplB2mV6i1dutSyTnidSHbu3Gm5fv36lrX7Wlr0fdUySd1vKC0T0Ul20+P777+3nFbHrkSif3O1atVcz+l7rpOia4fDUGzevNn1WNfX487nn39uOVZlOaEaPny4ZZ2UUsuU0ipfClaqp50jQ3kt728WfU5LBdeuXRtw24mgcuXKlr/88suAy+hvFe0s5n0u2PdMj9W1a9e2rMd5nZzVS/fVOuHzRx99ZLlv376Wn3jiiaCvFU96C4Z3n79y5UrL+jlq27at5WnTplmO5GTfhw4dsqxd7LQkdsaMGa51dILtZKSdd/V3tf5G8KJUDwAAAACSECdOAAAAAOAj8dugpJNOdHnXXXdZ/uabb4Kuo5PMJVp5nho7dqzlUMrzvOUnV199dcDl9HJ+pUqVLGsnFe0G5S2100v7Si9Jv/nmm0HXR+ooVaqU5ddee83y/v37LZ9zzjlR2bZ2nHKc5Ctv0G5DoZTn6cTTjuM4L774ouVQ9mPaIe3ee++1vGnTJt91vZo1a2ZZy7m0w1yieeaZZyx732/tEhhsvxmKtCb+/u677yxrmal+hxKBdml7/vnnLV9//fWWg5WDZpRO+Dp16tSgy91+++2WE608T2n3RJ1gVDuVXXzxxRnaxr59+yy/9NJLlkP9LmonPX1fK1asaHnBggWWhwwZYjlYp8l4uP/++y2ndauBvi/6GyuS5XlKb2HQTnqaU42WIqdVnpfouOIEAAAAAD44cQIAAAAAHylVqqeX8Dt27GhZu6V4SyYaNGhgWTueJRrtJOTTCdFxHHeXlnPPPdf1XL169cLa9llnnRUwawmf47i7GGlZh9qwYYNlnfjv5ptvDmtMSGxXXnll3LbtnYRTS0sS1YkTJyzPnz8/rHVLly7tehxumfENN9xgWUuIvF2yQplE84cffrA8bNgwyzrR5IUXXhjW+KJBJ11Nq/RLJ1HNyIS+WqLqOO5Ohsli3rx5lrX8XSed1RxJb7/9tmVvt77y5ctb1v9eiSwWpave43MotJRKO+npxOX6WV6yZInlRCrPU7Nnz7bs/e3UuXNny08//bTltEprER49tu3evduyHqe1y2Yy4IoTAAAAAPjgxAkAAAAAfHDiBAAAAAA+kv4ep507d1oePXq07/K33HKL6/Err7wS6SFFxddff21Z672D0ZnGo3VPwaBBg1yPtYX5rbfeGnAdbb37v//9z3Iq3OOk90DofSPa7jmz8LYETxT6mfPOIh9P2hL7+++/911e39++fftGbBz33HOP5aZNm7qea968uWWd7iEYvd+pfv36lnUf4DiOky1b7A9DEyZMsKzj0Xt3HMdxSpYsGashJZVQPqMZdfjwYctbtmyx7L1PRT//3vsb8X+OHTtmWac70DbljuM4P//8s2W9n1nvWdV9aKLeC7Rw4ULLek+W9/64cO9r0hbvJ0+eDPq6hQoVCnmsqeLAgQOW9beh3huprd1PnToV0uvqe6733fbu3dtyrH9vcMUJAAAAAHxw4gQAAAAAPpKyVE8v3WlJlHfW93/ky5fPsrf8JFls3rzZdxm91ByP1qDVq1cPOBZvK95UlT17dst33HGH5cxYqqffuXiUYgXz1ltvWdaSlXh76KGHwlq+atWqlqNV5lqsWDHXYy0RDrdsb+PGjZZDmU4h2o4ePRrw38uWLet6nJEW5Grw4MFBnytQoIDl3LlzR2R7kaLTR2h5nrekMRp0e1r22aJFC9dy3sfJRj+LWjqnpWBpKVq0qOUdO3ZY1ukDdOoP/XfvFCwvvPCC5SuuuMJyMpRAajni0KFDLaf1PgYrz9P3ceLEiQGzttDOmTOna/2uXbta1ltI9DdCqtEpHsaOHRvWulraqOV8juNuYd6vXz/LH374oWWd+qJatWphbTs9uOIEAAAAAD44cQIAAAAAH4lTQxMG7bbzzTff+C7/66+/Ws6bN29UxhRtWs4RzFVXXWW5YMGCURxNYFoy0KhRI8s6A7l67733LB86dMhynjx5ojC66NOSgJUrV8ZxJPF3zTXXWL7gggss9+/f3/L48eNd60SrvLRx48aWR4wYYfngwYOWk22/oDPex4qW7s2dO9dy5cqVLWuX02C0Q5rjOM7FF1+c8cGFSTs9KW/X1UjRUkWv66+/3vJ5550Xle1HQqzLtdq3b29Zyzu95WWJVt4YrsWLF1vW8uFQyvPTUrx4ccv//e9/LWs3UW9pajLT/fnq1asDLtOkSRPXYy2NHDlypGUtD9NuccFomaDjOM64ceMs6/fmscce832tZKW/UbU7a7BbaNSTTz5p2Xss1t9SXbp0sazfGy25fPPNN0MbcAZwxQkAAAAAfHDiBAAAAAA+kqZUTzuYaBepYB2atFQoWTuZ6CXitm3b+i6vly537dplOVoT4KalXbt2loOV6mnJzokTJ6I+pmjTv0Ev1Wd2Wg5x0003WX7ggQdcy5UrVy4q29cSM+3wuGrVKssNGjSIyrZTlZblejtK+ZkyZYrr8ZAhQyIyJj9aRrhp0ybLpUqVslykSJGobDutToJaYp1oLr30Ustr166N6ba1k553gtFUouVj9erVs6zH8FBNnjzZsnYQnT59uuVrr7027NdNBlqupeWvH3/8sWWdvNdxHGf+/PmWg33G9PtZqVKlgMvMmjXL9ViPM9qJT7vtJXJZbnrocUAnGM+oOXPmROy1IoUrTgAAAADggxMnAAAAAPCRNKV6PXr0sPzVV19Z1surOgHrkiVLLOfIkSPKo4sO7dIWSreqRKKd1JC5afmJdnvs1auXa7l33303KtvXrnq5cuWKyjYyM53sOa2JXhNVxYoVLZ911lkRe12dEFJLzb2SZV8Zi656y5cvtxysvFHLsFKNdggsWbJk2Otr2at2MNVyMS2X9pbtzZw503K0upxGi45XJ0StX7++5ePHj7vW0Ynab7/9dst9+/a1rN0Jg/nkk09cj/ft22dZJ9P96aefLKdaqV6keDsijhkzJk4jCY4rTgAAAADggxMnAAAAAPCRsKV63tIGvcSptGOeXl5N1vI8pROK6WSA06ZNi8NogMjSLkjRpN+jyy+/3PLTTz9t+brrrrOc7BNqxppOPBmKaHVP9KOTVGoZ3W+//RaV7WlnLS3d8brooouisv1k9P3331vWMvyWLVta1k5/yURvMdBOt2effXZUtqe/je6//37LOoGwlrE5juNcffXVlrUrX+nSpaMxxKjRcs7vvvvO8qlTp1zLael2KCV5odLPrpa4nn/++RHbRqpasGCB63Fa+8544YoTAAAAAPjgxAkAAAAAfCRUqZ5O+Hbbbbe5nvvss88s60RbL7zwgmWdGDcVnHHGv+e1OkFnKKV6rVq1svzBBx+4nsuTJ08ERvf/0kuqnTp18l2+W7dulrWcCqnrlltusbxu3TrXc9pFMlu2wLum7du3W/76668t62S23kv9OjGxlsuoJ554wvLQoUMDLoN/6cSR48ePD2td3TfFkn6mYjEpunZ29Zaeaym5TtCc2elkpdpVr1mzZvEYTobpbxo9hn/44YeWo1WqF4yWynonbv3Pf/5juU6dOpb1N0SZMmWiOLrIu/jii6Pyuhs2bLCsxyWvKlWqWC5RokRUxpIs9Bh/9OhRy3oM+eijj0J6rbJly1p+6qmnIjC60HHFCQAAAAB8cOIEAAAAAD44cQIAAAAAHwl1j9Pbb79tedmyZUGX05aZHTp0iOqYEoXWeF9xxRWWv/zyy4DL6+zLdevWdT03YsSIoM+FY/fu3a7HDz74oGW9/0Rp+89HHnnEsrbvROrS7+ukSZNcz+m9RXrP26JFiyzrDO1675K2nx04cKDrdbUd7Ny5cy2PHDnScvXq1UMZfsIYNWqUZb0XwXGi19568+bNlvU+suPHj/uuO27cOMvB7l+LNm1HfujQoahsQ+9ruvfee4Mu17t3b8vRugcjGQVrR16+fPl4DCfDFi5caFnvwU6Uv+eaa65xPdbxattyvR/5nXfesazH88zmjjvusJzWlAzNmzePwWgSl+53e/bsadl7/A+Ffm/0s3rBBRekc3TpwxUnAAAAAPDBiRMAAAAA+Ih7qd4bb7xhWUu3vK677jrLr7/+elTHlIjy589vWcte7rnnHss6Q7Zau3at6/GgQYMsB2uFmi9fPst6qVWzt+V4sPI81bhxY8up1pqzR48e8R5CwrvsssssaztRx3Gc559/PuA6jRo1sqxtR6tWrRowp0U/71qqF2+hlN+qjRs3Wva2A89Ia9atW7dafuaZZ1zPTZkyxbK3vXYg2tq4e/fulhOtLPfIkSOWdf/mOO624cF8/vnnlrUsR8t3tJTUcdwlK5mdTjWiWduRp4JkmHKjePHilgcPHmy5TZs2llesWGG5fv36sRlYgtB965o1ayx792m67+vcuXP0BxYBWoqt0/zo7Ry1a9e27J3S4bfffrOsJbd6nPVOjRMKLbXXEnH9rMYaV5wAAAAAwAcnTgAAAADgIy6levv377fcv39/ywcOHAi6Tp8+fSwXLVo0OgNLEjVq1LA8YMAAy126dLGcVtconZ29cuXKAZc599xzLWspS0a7UbVq1SpD6yeybdu2xXsICU9LTvVyfqzoZf9Eol1EtUteKGV7WrrrOO7Obnfffbfv+q+99pplLQHct2+f77pelSpVsvz4449bPuOM+P8/Ou28VLNmTcvLly+3/N5777nWadq0acDX2rNnj+X58+db1vI83U9PnjzZtX6RIkVCHXamkmhlnBmlv1UmTJhgWX8D6T4xkdxyyy2Wy5UrZ3n27NmWM0Opnu4f9Heoyps3r+ux/q4988wzozOwCNi+fbtl7Sy7c+dOy9rBtVatWpZz5szpei09nm/ZsiWscWiJs/eWE70dJVFu74j/0QwAAAAAEhwnTgAAAADgIy6levPmzbOsnTzSklYZX2bWunVry1oqFuyScqh27dqVofW1g5B2aNGuegD+j35fHnvsMcstW7b0XffkyZOux9rdMq1JWCNFy/O0a5KW+yYCLZlp166dZS3Fuf/++13r6GS9ixcvtjxt2jTL2mHw/PPPt9yrVy/LTHIbGu2klwpd9bSb4q+//mpZS0JvvfVWy4lQ0voP7ZqmpaWrVq2Kx3CiTm9J0E6lo0ePtqylpLo/0XI2x4lvx7dw6N+sxyAt1VMfffRRVMaht5x4jxuJeGtO4nxLAQAAACBBceIEAAAAAD7iUqqnlzizZs1q+dSpU5a1RMJxHOfHH3+M/sCS3F133WVZS2YWLVoUk+3nyZPH8syZMy3fcMMNMdk+4Ee7H+mks6GWDMeCdhiaOnWq5Q4dOsRjOEY7awUrJwxlwthE0LBhQ8v6mfjll19cy4VSWqzlVWPHjrUcSpkl3LQUSj9vmpNJ7ty5LWs5V8eOHS1rF7FHH33Ucry/S1qipt09Bw4cGIfRhGf16tWWtXOc7lsdx3FefPFFy9qd1NvZLZDevXtbDqV7aSLSLnWDBg2y3LdvX8uhdsjTz3r79u0tezuV/kPfvwoVKlj2/vZPRFxxAgAAAAAfnDgBAAAAgI8sPp1rot7W5tJLL7WspXr9+vVzLdepU6doDyWlHD161LKW7TmO47z//vuWvZNn+rnvvvssey/Z6yXWRJ3UL5q040zt2rV9l9FJOBF7WkKq3dBeeeWVeAwnIN0/792717KWhGmXUsdxd9ULhe5btRuUtzxKO3gmQzlFqLSD1IYNG1zPTZkyxfL69estFytWzPIDDzxgWbuoITQ6waWWTs2aNctyixYtYjqmaNPPVdeuXS2XLVvW8ogRIyx7P1daFp8R+pl2HMeZOHGiZZ2w96GHHrKsx/1cuXJFZByRpvtELRvzjnf37t2Wg02+XKZMGctdunSxrO9JKtCJ0wsWLGh5yJAhlnUSce8tGPp7Rjv0HTp0KOD2IvUZjrKAHwquOAEAAACAD06cAAAAAMBH3Ev1ACBWjh8/brlatWqWe/ToYVm7UwKIrvPOO8+yTiaspfupTDvWaQmudobbt2+fax3tCqkT6Gpns61bt1pesWKFZS3V/+2331yvW7p0acs9e/a03K1bt7T+hISj72n16tUt6y0MjuMuhdYuq9p9T8vztKQbmQKlegAAAACQHpw4AQAAAIAPTpwAAAAAwAf3OAEAAADAv7jHCQAAAADSgxMnAAAAAPDBiRMAAAAA+ODECQAAAAB8cOIEAAAAAD44cQIAAAAAH5w4AQAAAIAPTpwAAAAAwAcnTgAAAADggxMnAAAAAPDBiRMAAAAA+MgW7wEAANJv0qRJrsfDhw+3vGXLloDr1KxZ03KzZs0sFy1a1HLbtm0jNUQACahXr16ux88884zlxx57zPKQIUNiNSQksJMnT1resWOH67kff/zR8jvvvBNw/YULF1reuHGj5SpVqlhet26da50sWbIEfK277rrL8pgxYyznzZs34PKRxBUnAAAAAPDBiRMAAAAA+Mjy999/p/V8mk8C8NemTRvLenl6zpw5lkuVKhXTMWXEhx9+GDAPHjzYcu3atS0PHDgw4L/D35EjRywvWLDAcp8+fSzv3LnTtY6WUwSj+30thciePbvlMmXKuNZ56623gj6H/5d+16+99lrXc9OmTbPcsGHDmI0pGn744QfLa9euzdBrvfzyy5Z13/LAAw9YvvLKK13rNGjQwPJ5552Xoe2nqq+//try2LFjLU+dOtW13KlTpyyXKFHC8tChQy23b98+CiNEotq+fbvlu+++27KW3aUl2LEmlOVDXUePh6NGjQppXCEKuHGuOAEAAACAD06cAAAAAMBHUnbVO3bsmOW9e/cGXGbRokWWu3TpEvY29HJh48aNLQ8bNszyFVdcEfbrJov9+/db3rx5s+XXXnvN8nfffeda59NPP7WsHU+0c49e/s8s9FLzl19+afndd9+13K1bt1gOKSxaMuM47pI873N+/06pnr+DBw9a1u/Oq6++aln3T+ecc45r/VDK6HT9PXv2WNbysm+//da1zk033WR56dKllkuWLOm7vcxI94fe45S+58lIy0NvvfVWy95jQkboflPLy7z0cxlq+VBmoMfqRx55xPKuXbtCWl87cupvqPXr11vWDp5ITS1atLC8Zs0ay6GU0IXq5ptvtqylv47j7tYXzMyZMy1HuFQvIK44AQAAAIAPTpwAAAAAwEfSdNXbunWr5f/85z+WlyxZEnD5cDt5hLp+sWLFLGspxoUXXhj2NhLN7NmzLeuEd998843l9LyX119/veV58+ZZzp8/f9ivlSxmzZpl+fbbb7d8/PhxyxMmTLCcyKV6derUcT0OVoaXUaGU8dWqVct3mUGDBmV8MHG0cuVKyzVq1Ai4jH6munfv7nrummuuCWt727Zts6xdtvr37x90nUsuucSy7gfPPvvssLadag4cOGBZP6tHjx51Lacluzly5Ij6uCJNS2viXR6nx2T9LGdGWl5bqVIly8GO2/reOY57v1K1alXL2hm2UKFCllevXm35oosuSseIY+vQoUOux7qvXbFiheX33nsv4Ppa5vjzzz9bbtq0qWu5ZcuWWd60aZPlc889N8wRx4dOYNuyZUvLJ06csBzqb0EtN9fPmx5D9HaYjz76yLW+TmirZXtffPGFZd3X6r4pAuiqBwAAAADpwYkTAAAAAPhI2K562t3JcRxn9OjRloOV56WHXjocP368Zb28qGWCOhnYSy+9ZFk7jSU6vdzaqVMnyzrBpveSdiB6CddxHCdnzpyWp0+fbvnjjz+2PHnyZMs6qWGq0fdYy/OSkbc8LlqleqG8bijLpPVd1Ml4k62kr1mzZpanTJkSsde94IILLN9xxx2WdcJbx3Gcr776yrLun//666+IjSXZzZ8/37K+Xzqxq+MkZ3keEpN+zkKZTFl/83i/4zpRs3ZQU9oRUss0e/To4T/YGPnkk08sjxgxwrJ2W3Ycxzl9+nREtqffey/9zdO3b9+IbC/atFRRf8voLSzeDsnjxo2znJFyubp16wZ9rlq1apbbtWuX7m1kFFecAAAAAMAHJ04AAAAA4COhSvX0srH3su8ff/wRlW0WLVrUcv369S1XqFDBspbqqdy5c0dlTJGgl1dXrVrlek5L7IJNxKh/26OPPmq5SZMmlsuXL+9aR19Lu8rphMWZvaynZs2alrWDUSLzlrTp42DlbtoZJ1gnPG/3nGiVAKa1zUSnpRFz586N+vZ0f9i7d2/Xcx07dgw4Lvxr3bp1Af9dS0xSgZaNxrurXmakv4f0mPz7778HXF475GlJ1dVXXx2F0cXfyJEjLestCEWKFHEtp50Hr7zySst6zCpYsKDv9vQ3juO4O8Tu3r3bf8AJQMuvdULZYN3z9DPlOBHvZpfQuOIEAAAAAD44cQIAAAAAH3Ev1dMJ2+666y7LOpGg46Rv4tVQrF+/3vKTTz5pOZTLq1u2bInKmCJBL097u98pLcm75ZZbLGvHO72EnZYCBQpYfuaZZyzrBGa6jcwoV65clvPlyxfHkURGrDvT6WS86SntC2UC3XjTCSaLFy9u+ddff7WsE1QPGDAgYtv+7LPPLHsnZY7WPjiVaIlLKtMJUR9//HHLwcraHcdd/t65c2fLOmnynXfeGakhphTvxL46eXqwSX/LlStnWSdkPeussyI8usTToEEDy9pl1TtJr/5myQidMNZx3OV9Dz74YES2EW16q0WwWziU7gMyG644AQAAAIAPTpwAAAAAwAcnTgAAAADgIy73OGnrxrZt21rW+5rS0+723HPPtaz37mj9qbeF9vPPP2+5e/fuAbevtf1XXHGFZa2dTQQTJ060PHDgwKDLaa25thrX/xah8LahHTp0qOUNGzZY1lm8c+bMGdY2kol+rseMGRPHkSQ/vXcqo98zbQ0b63uy0qNMmTKWdYoGbQ8eqdp8x3Hf1/T5559bPnLkSNB19D3Nnz9/xMaSjNasWWNZ20Hr/XR6v0kq0PsztU1+ly5dLHunW7jjjjss631Nug7+dfz4cctdu3Z1Pffuu+8GXKd169aWZ8yYEZ2BJYGePXvGdHve+0zPO+88yzrFA1IDV5wAAAAAwAcnTgAAAADgIy6len/++aflw4cPW06r3W2w57SsZcWKFZa1FED9/PPPrsfaNjvYNkqUKGFZ24Cec845QccbD1oyoe0ktTTPcRxnyZIllvVvOHnypOWjR49a1jbIdevWtaz/Hb3rK/1vnMpy5MhhWVuQtmvXLh7DSTqRKs/zlqkmQ3leMNWqVbP88ccfh7SOlozu2rUr4DIfffSRZW07nlZ5ntJW8FqS1b9/f8s69lSmpeDZs2e3fOutt1rOli3uM39EjZavL1261HKoJZzeku9weY9vyUyPofp7xluap79VChcubPnhhx+O2Fj+97//Bfx33bYe8zKbr776KmB2HMcZP358rIeTYVqyHexWGf33Xr16uZ779NNPA66j09H85z//8R2Hlos7TvDpcLQEMtb7V644AQAAAIAPTpwAAAAAwEcWn+514be2C9PLL79suUePHpa13MRxgpfRaXlakyZNAq6vZSn9+vVzre+9LPiPZs2aWX7uuecsJ3KHFO10pOVxpUuXdi2nHbGUzvq+ePFiy8E6DHqdeeaZAbcxZcoUy9r5MNUcPHjQco0aNSx//fXXlvUzOn/+/NgMLElo+VedOnXS/Trp6ciZzIYMGeJ6/P3331ueOXNmwHVC/U4HE2z96tWrWw61tDDZXXTRRQH/3VsWntlt2bLFsn5m9TMaSqlozZo1XY/feOMNy4l8fA7FBx98YPmGG24IutwFF1xgWY/bkdSgQQPLWt4fi20nqtOnT1vWEjLv7Qi6D86aNWv0BxYBK1eutKy/X1R6jhvhruM9fgdbp3HjxpZz5cpluU2bNpZbtGgR0hjTEHDjXHECAAAAAB+cOAEAAACAj7iX6qn169dbrlixouu5YJfrdCLI4cOHW9bLjlOnTg26TS1j00nTtGwwWVx++eWWv/3224i9bqiXWq+99lrLOultKvvxxx8taxctLc9T77//vmUthYBbtCbA1Y57wUpWk03x4sVdj7dt2+a7jn6ndXLWRo0ahbRN7cC0fPnygMvoPvTZZ58N6XWTEaV6odHj0zfffBPWujpx+v79+13PaYl4srvqqqssr1271rJ2znMc93GkcuXKEdv+Y489ZnnYsGEBl3nttdcsd+zYMWLbTgavvvqq5c6dO1ueN2+ea7mmTZvGakgRo7ca3HnnnZbnzJljOZFK9YKto+XiEfgdSqkeAAAAAKQHJ04AAAAA4COhSvWUt1Ru4sSJYa2vf9d5551nWS9FO47jtG/f3rJ2pUtGe/futaxdXd566y3XcloyoRO3NW/e3PIff/xhuWXLlpb1sqm3nFI773hLC1KVTnR33333+S5/4403WvZOagh/6ekAF0yqdN/zloUsWLDA8llnnWVZuxBdf/31ltu2bWs52MThXocOHbKspX47duywrCWEmzdvDul1k4V2xNQOrLpv/fLLL2M5pIRz4sQJ1+PLLrvM8g8//OC7vpbn6YTi3i6Sya5KlSqWtcRbj6HeDqzRmlz66quvtqylgkpL9Tp06BCVcSSS48ePW9b3XTsEa+mk40T2OBUP2t1Sjw862XckS/X0O+A9LgfrfB1sGzop87hx4yyHMvluAJTqAQAAAEB6cOIEAAAAAD6yxXsAwfTv39/1eMKECWGtr5fubrrpJstdunRxLaeX9ZJdwYIFLWuHO81p0Unc2rVrZzlY2aN3MuHMUp6nunXrZlk76+hEhkonzUP49LOonfd0kmudSDctWjaQzGV7r7zyiuvxpk2bLGfPnt1yJLtv5cmTx3KyTPAYSTrxp36OtNNWZjdy5EjX41DK85SWnqfScdpx3OXz2gFXj69du3a1HK3SPG/nPC3dx/+59957LWsp5YoVKywne2meV+7cuS1rmageZwsVKuRa55JLLrGsk9A+8MADlmvVqpWhcW3cuNGylpXu27fP8rFjxyzr77B0luoFxBUnAAAAAPDBiRMAAAAA+EioUj29ZL1w4ULXc3opNG/evJZPnjxp+a+//gr4utq9TEssHMd9eTEz0u5YY8eOtazdU3LlymVZJ0MLtQQwlWmZ0hlnBP7/EHqJeOjQoVEfU2ahpXrKW6qnE+gGK+OrU6eO5WXLlmV0aDHlLZnwPo4ULYfQ/cOff/4ZcPlUnhzz7bffDvjvFSpUiPFIEsuGDRsse0tIQ5E/f37LqVb+9NNPP1keMGCAZe0+WL58+YDLRJKW5z3++OOu57SDnJb+t27d2nIyTu4aLv0tqeVe+vmsVKlSTMeUCLTrqpbgOY77t/Tq1asta5fXjJbqlSlTxrJ2wdZJsWOx3+CKEwAAAAD44MQJAAAAAHxw4gQAAAAAPuJyj9OePXss33///ZZnz55tWVsKOo7j1KtXz7K2Of3iiy8s9+jRI+D6u3btsvzLL7+4Xjez3+OkNc6jRo0KuMyrr75qmfuawjdr1izL48ePt5wZ2zjrPUZptQ0Pdv9SKGrXrh3S9oP9u+a0XivV6T1NjuM4DRo0sPzZZ59Z1prykiVLWm7fvn3Uxpao6tevH+8hxJweg1u0aGF5y5YtYb/Wk08+aVlb6aeCRYsWWQ7Wml2nt4gkvYdz5cqVlvWeJq+GDRtanjhxYlTGlai0Bbf+ZtT75fVe+8xC71tcunSp6znv439E8p6j559/3rL+rg9G74mKJK44AQAAAIAPTpwAAAAAwEdcSvU+/vhjy9rqUS8bV6lSxbWOthS+8sorA+ZNmzZZ9s5a/o9169a5Hmv5SWYxb948y08//XTAZQoXLmxZW5EifFry9Pfff8dvIAkgrdbgAwcOjPFo4LV9+3bLjRs3dj331VdfWdbPcdmyZS1rKUuJEiWiMcS4OXLkiOWDBw9avummm+IxnISxYsUKy+kpz6tYsaLl9LQr1nbeP/74Y1jrFilSxLK2Wo6lGjVqWNZbEkJ16tQpy1pC+9///tey/jfSKVy8mjVrZvnZZ58NeyzJTFuQ63GqQIEClr2/S+HvxRdftKz7zWC3yXh/I2k7808//dRyWmWm/2jUqFHI4wwHV5wAAAAAwAcnTgAAAADgI2alet9++63l2267zbJebqtWrZrlJUuWuNY/66yzfLdRqFAh32WqVq3qu0yqu+eeeyxrmYNehg7WIQXh03KQWMxqnWhC7aQXrQ52WnaRSCZNmmR52LBhlrX8+O23387QNrS8TEseli9fblk7SOn+2Nu1SD+7Wr77xBNPWE618jz1008/WdaSb+0Mi/Dpb4MHHnjA8uWXXx7S+ocOHbL8zDPPhLVtLbOcM2eO67mcOXOG9Vrp9fPPP1tes2aN5fPPP9/y9OnTXeusX7/espbevfPOO5a15CnYcadp06aux5MnT7ZcsGBB37GnEt3fbdiwwbL+ZgzlN2Yqq1u3rmUtwXMc97FG6a0K2hUvGG+pXii/mXSdG264wfI111zju256cMUJAAAAAHxw4gQAAAAAPmJWqqeTq+rktDVr1rSspSShlOZ5ffTRR5Yze/eyP/74w/Jdd93lem7v3r0B19HLsHny5InOwDKhsWPHWs6RI0f8BhInoZbgBSupC2V9LQH0vk5a5YH/WLZsWVjbiwQtMdq2bZtlLZHRySpDNW7cOMvaZezzzz+3HKyMJ1++fJarV6/uet0mTZpYfvjhh8MeV7LT0kqlJeaZUdGiRS3r5+fAgQNhv5aWmmmOJB2jdvlKq9tcJOjkyNqpTTtZaplRJOnxXCfDfe2111zL5c+fPyrbT1SnT5+23Lt374DL6C0M+ts1VqWcieTmm2+2PGHCBNdznTt39l0/PbcqhLJOpUqVLL/00kthbyNcXHECAAAAAB+cOAEAAACAj6iW6mnHNu2soZfeGjZsaFnL83Rdx3F3kVFTpkyxrOU2uo3M2MnsjTfesKxds7w6duxoeciQIVEdU6pLT3lpZqPfUS0ZcZzQu+9FSqxK8oLRCWa1M90333xjWSfHVKF0zErLhRdeaLly5cqWtUOc979PZqSlOYsXL7asE5VqCWNm1LJlS8tPPvmk5VWrVsVjOEa7mfbv3z/gv7do0SJm4ylXrpxlLdXT30bpod//vHnzWtZ9xOzZsy1ryWBm9/HHH1vWroraeVg/05mxPC+Y5s2bB33u3nvvtXz48OGwXtfbTbNw4cKWtduzlg1qtz4tHY4WrjgBAAAAgA9OnAAAAADARxaf7nMZak2nZQ5azqCT2xYrVsxyhQoVAq7rOO4JG0Ohf5d20fFO7KqTTSa7zZs3W77xxhst68SNXtdff71lfS8GDRpkWd8/BPfXX39Zzp07t+UZM2ZYbtOmTUzHlExiUVKr5XlaNhhv2glo+PDhlrds2RJwed2/DRw40PVcKJ2xevXqFeYIMyctm9QSkg8++MCydiPN7PQY1LNnT9dzegzXbnYZpZ1KtQxNy18rVqwYse1Fgn6udMy7d+8OuPy5557reqzf3+zZs1sO1hkO/8f7e1e/u/o7SbuZ6kTECM2OHTss6+/9hQsXWp45c2bAdb2/0XUSW/1+aEle1qxZ0z/YtAX8UcIVJwAAAADwwYkTAAAAAPiIaqmedtPo16+fZe3wopO/uTbsGVe4ZTw6WaGWo11yySVhvU4y0c5BWqaQFi0/GTZsmOVGjRpFbmCZBKV6kaNd9YJ1d9Oyu1q1agV9LV0u3p30kFzatWtnuXz58pb79OljOVeuXDEdU7LSCW21VE8n0uzevXvYr6udztLq9AXs3LnT9Vg7LL7wwguWu3btGrMxIaFRqgcAAAAA6cGJEwAAAAD44MQJAAAAAHxE9R4nAAAAAEgy3OMEAAAAAOnBiRMAAAAA+ODECQAAAAB8cOIEAAAAAD44cQIAAAAAH9niPQCkvtOnT1ueOXOm5ZEjR1pu2LCh5SeeeCI2AwMAxNyvv/5quXbt2pa3bt1q+ffff7dcqFChmIwLAPxwxQkAAAAAfHDiBAAAAAA+KNVDVJw4ccLysmXLLLdr185y1apVLQ8cODA2AwMAxNW0adMs//zzzwGXGT16tOURI0ZEfUwAEAquOAEAAACAD06cAAAAAMAHJ04AAAAA4IN7nBAxv/32m+UOHTpY1nucihQpYvm9996znDNnziiPLvo++ugjy+PHj7c8e/bsgMvfeuutlu+9917Xc7Vq1Yrw6BKf3hf3+eefWw72/mkb40aNGkVtXAAi66WXXvJdplSpUjEYCQCEhytOAAAAAOCDEycAAAAA8JHl77//Tuv5NJ+MpmAtSh3HcS666CLf9VeuXGl53rx5lrdv3+5abtasWZYrVKhg+d1337XMrOWBbdu2zfW4Tp06ljdt2mT5iiuusLxu3TrLWbNmjd7gMki/F/pZ8HrhhRcsL1++3PK+ffssZ8mSxXd7+fLlcz3WMrQXX3zR8jnnnOP7WsnC+1389NNPLbdp08Z3/XPPPdfyjh07IjcwpDz9fmmJrZYSO47j/P7771HZvh5rypQpY3nYsGGWy5cvH5Vtx8sbb7xh+c4777R89OhRy7qvXLx4seV69epFeXQA8P8I+OONK04AAAAA4IMTJwAAAADwkbCleqH66quvLGunHi1v0m5dxYsXd62vZQJq69atlhcsWGC5Ro0allOpbCot+h69+eablh966CHXcvv377fcoEEDy9phrkSJEtEYYkRoV8CpU6da7tevX9ivlT9/fsvnnXee5WPHjlnesmWLZe/3UEtWFi5caPnGG28MeyyJqnXr1q7HwbrnBZM9e3bLDz74oOW1a9e6ltOSp5tuusnyDTfcENb24kFLlvVzEKp33nnH8qpVqyzXrFnTcrD34fbbb3c9LliwYNjbT1Qffvih5YcfftiyHivSo379+pb37NljOXfu3K7lZs6cGXC5AgUKWJ4/f75lPe4kq2bNmlnWv0117drVspZBI3xz5861rJ/xH3/8Meg67du3t/zcc89Z9paSZyaHDh2yPGDAAMt6m0fZsmVd68yZM8dy3rx5ozg6RBmlegAAAACQHpw4AQAAAICPpCnVO3XqlGXtzqMTrWp5U548eSz37t3b8v333+96Xb2c/cADD1hu166d5YkTJ1oeM2aM5T59+oQ8/mTWs2dPy+PGjbOcLZt7/mT976KTuyYLnURVJ+dND530V8uitNteixYtLGvpkOOkVqmelptpKYiWjjlOaN0Hle67wl3XcRznqquusjx9+nTLoXTtjIRvvvnGsu5jtLzur7/+svznn39GbNuhvHfatdBx3F0zX3/99YiNJTPQ/YHjOE7Lli0t6z5BJwLXz0HdunWjN7gYCaVUT481zzzzTNTHlAqOHz9uuWHDhpaXLl1qOT37R+3w+Oijj6ZzdMlDf2Nqh1f9janl9WnRCe31VoVEpvt3/T2iHUhr1apledCgQRHbdlqvNXjw4ID/rvtUHWOEUaoHAAAAAOnBiRMAAAAA+EiaUj2d0Pa6666zrOO/5557LGvZnU4w6KVd+SpXrhxwmWLFiln+4IMPLJcrV85v2Enls88+s/z4449b1gmEK1asaHns2LGu9fVSbzIaOXKkZS1NuPzyyy3ffffdrnW8j/1oWU7z5s0t6yScjpP8pXq//PKLZX0vtZtYWp0EQ5HRUj1d/5JLLrGsZZolS5YM+3VDpd0W//jjj6hsQz+7pUqVsqx/++bNmy1//fXXQV/rsssus/z+++9bzizdRUOh3Viffvppy97Sxt27d1vWz66Wf48aNSoaQ4wZfS8cx3281BJULfnW8t0qVapEcXTJ7fDhw5arVatm+fvvv7es3/Gzzz7bsnaG85ZIaWdc7Qz75ZdfWk7kzrjh0v3u8OHDLet3Nz30Oz1p0iTLXbp0ydDrRpr+9w9WEpcMfM5jMoJSPQAAAABID06cAAAAAMBHNv9F4uPgwYOuxzoZo16W69Spk2XtTBWq0aNHB3zd888/37Je2k6F8rzTp09b1r9fS6p0mfvuu8+yvheFCxeO1hDjQrsvaldALVnI6N+sJTrLly8Pupx2r7n++usztM140MmoZ8yYEXAZ7+V1fZ+11EHLG9NaP1y6vk4KqeW70SzV0wmkFy9eHHAZnbxXO32GSstqvF3y/rFz507Lzz//vOUhQ4a4ltMyPu0ImAod30KhE2Fq1y2duFnLanVCbS/9b6Gfg1Tq1NqmTRvXYy3PU61atbJMeV5gWkLnOO5Ob1qeV6hQIcu6H8uRI0fAZbRLsHc7Oln7kSNH0jPshOMtia5Xr57ltMqUw6XHFp0oNxFK9bRjXrDyvGBd6rzdfzMrrjgBAAAAgA9OnAAAAADAR8KW6umkmY7j7tKlZTyhlOft3bvXsrdbinb50rKcCRMmWNZJ5ZKRd+JMfQ90krsCBQpY1vK8bt26WU618jx15plnWi5dunTEXlc7IIXarUfLBnPnzh2xsUSTluKEMvmivt+O456cWrsHNmjQIOA2dH0td/ruu+9cr6tlcMHKhXSMWq6mk3ZG2oMPPhgwx1NaEz9rGVWwDqTJ7ujRo67H+n5oWfOKFSssB/t8a9dEnVzbcdzdrC688MJ0jTURaYn3pk2bQlrnlltuidJoUscLL7zgeqyTI+vnTLvf6b8rLeE7cOBA0G1ecMEFli+99NKQx5potMT2iSeecD0XSnle3rx5Ld92222WV69ebVnfUy/9Laul+vHqRpqRcjstQfS+TrglgHo7gvJ2GA423oEDBwYfaJRxxQkAAAAAfHDiBAAAAAA+OHECAAAAAB8Je49TqLQmPVeuXJb37Nlj+eabb7asM5M7jvu+pgULFljWNsDJSO/ruvjii4M+p/c16d9fvXr1iI1FW4Bqa2nvuFJVr169LE+ePDngMt5632RsQa7141r/Hczdd9/teuydxf4fl1xyiWWtST/77LMt6716XvpcKHXRn332meVRo0ZZfvjhh33XTUZ6n2ha/93KlCljuWDBglEdUyxp3X7Pnj1dz2lb/WBq1qxpWWv4tWV0JO+ZTGRz5syx7G39rPS407p162gOKWlt3brVsndqgGzZ/v3pplOEBLuv6ZNPPrGs77d32helU3Iks3Hjxll+6qmngi6XPXt2y3rM7t69u2Wd3kGPE97fS8ePH7e8ceNGyy+//LLlvn37+g09bvT3SLDjsrdluT4Otk4ovOsmYgt0rjgBAAAAgA9OnAAAAADAR8KW6l100UWux/Xr17f8wQcfWNbykeHDh1vWkigtP6lRo4brdSdNmmS5XLlyGRhxfJw8edKytoDWMijv5fjrrrvOspZXhVKep+UDWpbhOI6zfft2y9oCWssmtVTvnnvusawlUalA25POnz/fspYFqXvvvdf1OH/+/NEZWIRpe+9wL6k/9thjIS2nrXC1VE/LdKdOnWpZ9xWO435vtb345ZdfHnB7+nkdOXKkZe+s74UKFfIde6KaNm2aZf0bVd26dV2Px44dG80hxY220Nb9qeO4v4f79+8PuP7y5cst79q1y7KWhXuPZ1oGddVVV1nOkydPqMNOaqlSBhZNWjp/5MgR13O679Hj6G+//WZ5xowZlvv162dZy8i8tFytU6dOYY44MU2ZMiWk5VauXGn5yiuvtPzmm29a1n1ljhw5LKf1niaLSJXaZZS3HXki4ooTAAAAAPjgxAkAAAAAfCRsqZ7O1uw4jjNz5kzL2vHjm2++sdy1a9eAr3X++edb/vjjjyM1xISgZUgvvviiZf2bFy1a5FrHW8oUiM52rV1pJkyYYNl7efrcc8+1fM0111jWGch1xvN169b5jiNZafmOdpfSz7WWPiVr6cqSJUssr1ixwnd5LaENdeZ07UTUrl07y/pd/vbbby1nzZrVtb6uo93gdCzeMrx/aGnpqVOnQhpvotLSxo4dO1rOkiWLZS3V0e+64zhO4cKFozi6+NHPi7frpXZSnDVrVsD1f/rpJ8vff/+95Zw5c1r+4osvXOvMnTvXspb8DB061HL79u39hp60Qv3uB/L777+7HmspvmZ9/7Qz55lnnpnubceSdg31OnDggGX9PaTd87QsXL/jaRkzZozlsmXLhrROoktrv637tGAdCbVEVzuQhko7N99+++1hrx9pWoYXz5I8pWX+aZX8azmht6tfLHHFCQAAAAB8cOIEAAAAAD6yBOvy9f9L88l40U5euXPnthzscnSFChUsawmV4yTnRI5aHtWgQQPLl112mWXtqJNWt8AtW7ZY1m6FgwcPtqyXqnUbLVu2dL3WnXfeaTlfvnyW//zzT8tt27a1rKVW2hUtWcvWdu/ebblOnTqWN2zYYFk7F6ZC2ah2BAtWeqklS/p5y0i5TqRp6Zp2nFM6EaLjOM748eOjOqZI0PI8LdH99NNPLet+U//22267Lcqjy7z0e6DHJy1/1lKzYBNnJxotZ2zVqlXQ5XTycz3uXHjhhZa1PF9f1ztJs3aSC0Y7demExYlMOzS2aNHC9dyXX35puVSpUpZ1f6xddvXzpvR47jju/YL+tkpm+htJP2teetuDHr+fe+45y8uWLQtpm2ec8e81ienTp1vW3z/4l5YM6m9PL33/Y1SqF/CkgitOAAAAAOCDEycAAAAA8JGwXfW8Dh8+bPmhhx4KuIx2gdm7d6/l7777zrK3M9Jdd90VqSFGlZbn6WX71q1bW3711Vcta3mUl3beadKkiWUtyevQoYPl/v37W9buRGnRTnI9e/a0rBPjateuZC3PU48//rjl9evXW9ZSKH0vU4GW5wUrlX3llVcsJ1J5ntKxB/s7vB2VkqFUT79vWoajJdpVq1a13Lhx49gMLJMrUaKE5ddff92yTtCs35tu3bpZrlatWpRHl346fm9XOC3Z3rRpk2Wd4Fq7OgabcDij49IS6YoVK0ZsG5GmXWr1mB2qUPZpCxcudD1OlfI8pRN5p1WqpyV5mtNDu7RSnhc52nGPrnoAAAAAkMA4cQIAAAAAH0lTqvfUU09Zfv755y3rRK+rVq2yrF3atFRs9uzZrtdNllK9UaNGWdbJ77QbWLDyPJ0c1HHc3Zp0gs/Ro0dbfvDBB33H5J1Y7v3337fcr18/y9oBqGHDhpaTsTzvl19+cT2++eabLWt5npZC/fjjj5ZLly4dvcElqPLly8d7CL7uuOMOyzph9J49e+IwmoyZM2eOZZ1wMVi5Tq9evSxrN0zExokTJywH+2/01VdfWU7kUj2dXLZevXqu5956662A62iXXM2RpK+rx89UoxPYKj0e6YT2xYoVi/qY4k27g+pvRMdx7+v1e5hR3o7DSJt2vUwGXHECAAAAAB+cOAEAAACAj4Qt1dOOa47jOAMGDLCs5Xnbtm0LuL6WqOiEWl9//bVrOe3ckz9//vQNNkq0C5Z2gxk+fLhlLX1TWtrRuXNn13PHjh2zvGbNGstVqlTxHdPatWstaxc5x3GcefPmWdYJDkeMGGH54Ycf9t1GIvOWeurktlpm06lTJ8vFixeP/sCQIVqyqpOQKp2oNJFoKajjuD97wf4W3W8WKVIkOgNDSObPn++7jE5enCzeeOMN12M91mpZdyyULFnSsk5Cngo2b95s+dFHHw24jJb0J8vtCZGi/+31N4rjuG8j0M+k/n7R9zfYLQzXXnut67FORIzAtEOe5rToRLnxxBUnAAAAAPDBiRMAAAAA+ODECQAAAAB8JNQ9Tnof09NPP+16Tu9ZCmVW52zZ/v3T8uTJY1lnL3ccxzl58mTY44wVvX/m9OnTlvPmzWtZ24yquXPnWv7iiy9czy1dutTyFVdcYXnXrl2WZ82aZXn69OmWP/vsM8veduQ1atSwPGPGDMt6T1qy0Pd18uTJlvV+OS+d6V3v5dIWvZnRfffdZ3nBggWWzzrrrHgMxxw+fNiy3jd48ODBgMuHci9KPHhbEB85ciTgchUrVrQcrTbE2vb53Xfftaz3VOnnIbM4evSo67Hei6L3AhUoUMCy3kNaokSJ6A0uSrJmzep6HGy6jEi65JJLLHfr1s3yPffcE/Vtx5K2zn7ssccs6++ZwoULW540aZLl7NmzR3l0yUN//2hWderU8X2dLl26uB4XLFgwI8PKFNL6LaUGDhwY5ZGEjytOAAAAAOCDEycAAAAA8BH3Ur2RI0daHjp0qGXvZVO91JwrVy7f19XLe1u2bLHcq1cv13KFChUKdagx17hxY8vnnHOOZS110TI6Lb+ZM2dO0Nd94oknLC9evNh3HLptbcfZvHlz13KJPKN9RnTt2jWk5fr162f50ksvzdA2169fb1lboGtZRiLQ8tohQ4YEXGb58uWWH3nkEcs33XSTa7l69epZDuU7HiotH1uyZInlJk2aWNZW8jly5LCsrXu1rW28ffvtt5a1rNbrsssus7xw4cIMbVPLnLX8V/dB2t5Xp0TQUshUpuWf+t/F+93QFsf62Xvttdcs6+cT/9JW0Y7jOIsWLbKsx/NULpfS3zSvv/56wGX02E55Xng2btxoeeXKlQGX0eNB27Ztoz2klBNqC/LatWtHdRzpwRUnAAAAAPDBiRMAAAAA+IhLqd7evXstP/nkk5a1+5p31vFQSndefPFFy9oJrXTp0pYfeuih8AYbR0WKFLGsJVpvvvmmZb2krB3v0qKX8LWzWZ8+fSy3bNnSspYAaqeeVLNv3z7LWoYYrHOh47g76elyrVq1sqyldsF4t6HlO0pLUHUdLet85513fLcXKVp6p90aV6xYEXD5iRMnWp4wYYLrOe3KWKpUqYDrV61a1XLNmjUt79692/LUqVNd62hZlI5L32PN+t/u2WefDTiOeNPvvX5uvbTUsGjRogGXCdYpc+zYsa7lfvjhB8takqe0JEhLr1OtlEXfMy0j13JI7STopcck/R40aNAgUkNMWQ0bNnQ99pbupaIDBw64Ht9yyy0Bl9N9YrAucQjs+PHjljt06GD52LFjAZfXWybi3SE2lVGqBwAAAABJiBMnAAAAAPARl1I9LeHQEpsXXnjBcrly5UJ6rUGDBlkeNWqU5WuvvdbytGnTLAcrV0l0ellY8++//25Zu1h5O5YFoyWQlSpVsqzduDKL+++/37J2ggtWNuc47s+vdmzUMrq01g8mlHUaNWpkWUvYYkk/P1pypJPbpdXhUWkZXbBSP/0uZ/Q91u552t1TJ9tOVN7SnWC0jOyDDz4IuIyWl+nEmmkpXry4Ze0u1bdvX8s33nhjSK8VK1qKoxOKa4dAb3md7gfeeusty1puHuw908lYu3fv7nquffv2lhO5s2sk6TFp2bJlloNNOK0Tj2rH0ssvvzwKo0s8+rnSCZMdx911VScWTtRJupOBfvfXrFkTcBk9Zmh5PEITymTCiVia58UVJwAAAADwwYkTAAAAAPjIklbHMMdx0nwyvSpXrmy5bNmylmfMmGFZJ610HMeZO3euZZ1YUMuAOnXqZHn06NGWdQJXIBjtQKafq7S6lgWTP39+y+edd55l7UrYv39/y6F21VOJVgqltPxmwYIFlp966inL69atc60TbrldqKV62sGsevXqlh9++GHL5cuXD2vb8fbHH39Y1pJNxwmtu2Yo751+bh3HXT6tk25q989EoKV32oFUu65qqV16ZM2a1XKFChUsazdSLS874wz+HyXCo91RmzZt6npOP3/6OdNbFxAePf6/9NJLAZfRMm4tG0doQjnGaxmv48S9dC/ggNmbAwAAAIAPTpwAAAAAwEfcS/XKlCljuXXr1panT5/uWkc7QulkYzqh5s0332w5W7a4NAxEitBuWtoJRstyHMdxWrRoYVm7RerlZp2UMLM7fPiw5UOHDrme08v4WmL166+/Bnyt3r17B1zXS7tO5cuXL/TBJokpU6a4Hmt3yGB0v6/lPTqB7W233eZap2DBgukcYeR5PxNjxoyxrN1ZtZOelmP++eeflrUzqZaOO47jFChQwLJOiqnlI8lW5onEtWfPHsv169e3rB1zHcdxrr76assrV66M/sBS1E8//WS5SpUqlvfv3x9weS0DZoLh0Hz44YeWg3XV0/2pt1QvzijVAwAAAID04MQJAAAAAHzEpZ6tbt26lp9++mnLOullq1atXOtoBy4t7wOiQcvrTp06FdI6dDTyp2W2mr169OgRi+GkhI4dO6b5OBX16dPH9XjXrl2WdaLVAQMGWK5YsaLlY8eOWd68ebNlb6meTngJRJt2atPyPO++UifsRvpt2bLFcrDyvEsvvdQyZbnh01K9YGrVqhX9gUQQV5wAAAAAwAcnTgAAAADggxMnAAAAAPARl3bkAAAAmZ22INf78Hbu3Gm5bdu2rnVef/316A8sE+jevbtlndpGTZ482XLnzp2jPqZUo/d+Dx482Hd5bU3uOHFvT047cgAAAABID06cAAAAAMBHXNqRAwAAZHaFChWyvGPHjjiOJPOZMGFCwAykhStOAAAAAOCDEycAAAAA8EFXPQAAAAD4F131AAAAACA9OHECAAAAAB9+XfUCXqYCAAAAgMyEK04AAAAA4IMTJwAAAADwwYkTAAAAAPjgxAkAAAAAfHDiBAAAAAA+OHECAAAAAB//HxfwQbOWlH8mAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1080x324 with 30 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.figure(figsize=(15,4.5))\n",
    "for i in range(30):  \n",
    "    plt.subplot(3, 10, i+1)\n",
    "    plt.imshow(X_train[i].reshape((28,28)),cmap=plt.cm.binary)\n",
    "    plt.axis('off')\n",
    "plt.subplots_adjust(wspace=-0.1, hspace=-0.1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "85bc450c",
   "metadata": {
    "_uuid": "1e61e07d14b9b012748fdaac9eaf02e5263a475e",
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:08.808396Z",
     "iopub.status.busy": "2022-04-19T13:17:08.807626Z",
     "iopub.status.idle": "2022-04-19T13:17:08.815360Z",
     "shell.execute_reply": "2022-04-19T13:17:08.814644Z",
     "shell.execute_reply.started": "2022-04-19T13:08:54.317544Z"
    },
    "papermill": {
     "duration": 0.062666,
     "end_time": "2022-04-19T13:17:08.815592",
     "exception": false,
     "start_time": "2022-04-19T13:17:08.752926",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "datagen = ImageDataGenerator(\n",
    "        rotation_range=10,  # randomly rotate images through any degree between 0 and 360\n",
    "        zoom_range = 0.10,  \n",
    "        width_shift_range=0.1, # a float number, that would indicate the percentage of width or height of the image to shift. \n",
    "        height_shift_range=0.1)\n",
    "\n",
    "# datagen = ImageDataGenerator(\n",
    "#         rotation_range=10,  \n",
    "#         zoom_range = 0.10,  \n",
    "#         width_shift_range=0.1, \n",
    "#         height_shift_range=0.1,\n",
    "#         horizontal_flip=True,\n",
    "#         vertical_flip=True,\n",
    "#         brightness_range=[0.2,1.1],\n",
    "#         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8b03ea8b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:08.924647Z",
     "iopub.status.busy": "2022-04-19T13:17:08.923876Z",
     "iopub.status.idle": "2022-04-19T13:17:08.926749Z",
     "shell.execute_reply": "2022-04-19T13:17:08.927231Z",
     "shell.execute_reply.started": "2022-04-19T13:08:54.325340Z"
    },
    "papermill": {
     "duration": 0.059653,
     "end_time": "2022-04-19T13:17:08.927363",
     "exception": false,
     "start_time": "2022-04-19T13:17:08.867710",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 28, 28, 1)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape # (42000, 28, 28, 1)\n",
    "X_train[25,].shape # (28, 28, 1)\n",
    "X_train[25,].reshape((1,28,28,1)).shape # (1, 28, 28, 1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd0bca7b",
   "metadata": {
    "papermill": {
     "duration": 0.033646,
     "end_time": "2022-04-19T13:17:08.994038",
     "exception": false,
     "start_time": "2022-04-19T13:17:08.960392",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Each row of the total 42,000 rows represent a single image - now take a single one from them. lets say the image at row-index number 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4edd53ae",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:09.060804Z",
     "iopub.status.busy": "2022-04-19T13:17:09.059994Z",
     "iopub.status.idle": "2022-04-19T13:17:09.065443Z",
     "shell.execute_reply": "2022-04-19T13:17:09.065879Z",
     "shell.execute_reply.started": "2022-04-19T13:08:54.335975Z"
    },
    "papermill": {
     "duration": 0.03882,
     "end_time": "2022-04-19T13:17:09.066010",
     "exception": false,
     "start_time": "2022-04-19T13:17:09.027190",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28, 28, 1)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[25,].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "270246ef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:09.133104Z",
     "iopub.status.busy": "2022-04-19T13:17:09.132480Z",
     "iopub.status.idle": "2022-04-19T13:17:09.135146Z",
     "shell.execute_reply": "2022-04-19T13:17:09.135534Z",
     "shell.execute_reply.started": "2022-04-19T13:08:54.346104Z"
    },
    "papermill": {
     "duration": 0.03853,
     "end_time": "2022-04-19T13:17:09.135671",
     "exception": false,
     "start_time": "2022-04-19T13:17:09.097141",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28, 28, 1)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train[41999,].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "834e61f4",
   "metadata": {
    "papermill": {
     "duration": 0.031725,
     "end_time": "2022-04-19T13:17:09.198264",
     "exception": false,
     "start_time": "2022-04-19T13:17:09.166539",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### First see what this 25-row index image is\n",
    "\n",
    "So below plot is before Augmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b3ebea69",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:09.285547Z",
     "iopub.status.busy": "2022-04-19T13:17:09.284988Z",
     "iopub.status.idle": "2022-04-19T13:17:09.463438Z",
     "shell.execute_reply": "2022-04-19T13:17:09.462829Z",
     "shell.execute_reply.started": "2022-04-19T13:08:54.355343Z"
    },
    "papermill": {
     "duration": 0.234338,
     "end_time": "2022-04-19T13:17:09.463555",
     "exception": false,
     "start_time": "2022-04-19T13:17:09.229217",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f78a16b1390>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAANAUlEQVR4nO3db4hd9Z3H8c9nsw2iDSaaMYY0u6nFJ7LBNF7iQqVmqRYVNPaJNA9KFspOhQgtVtjoCk2exWVtKLgU0zU0Waql0LgZ/FsNEU2Q4lVijA5b0zBSQ8xM8EENxGR1vvtgTso0zj135p5777nJ9/2C4d57vufO+XLIJ+fc8zt3fo4IAbj4/U3dDQDoD8IOJEHYgSQIO5AEYQeS+Nt+bmzx4sWxYsWKfm4SSGVsbEwnT570TLVKYbd9m6SfSZon6b8iYmvZ+itWrFCz2ayySQAlGo1Gy1rHp/G250n6T0m3S7pO0nrb13X6+wD0VpXP7GskHYmIoxFxVtKvJa3rTlsAuq1K2JdJ+tO01x8Wy/6K7WHbTdvNiYmJCpsDUEXPr8ZHxPaIaEREY2hoqNebA9BClbAfk7R82uuvFMsADKAqYX9D0rW2v2p7vqTvShrpTlsAuq3jobeI+Mz2fZJe1NTQ246IeLdrnQHoqkrj7BHxnKTnutQLgB7idlkgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEpWmbLY9JukTSZ9L+iwiGt1oCkD3VQp74Z8i4mQXfg+AHuI0HkiiathD0u9sv2l7eKYVbA/bbtpuTkxMVNwcgE5VDftNEbFa0u2SNtr+5vkrRMT2iGhERGNoaKji5gB0qlLYI+JY8Tgu6WlJa7rRFIDu6zjsti+zveDcc0nflnS4W40B6K4qV+OXSHra9rnf82REvNCVrgB0Xcdhj4ijkq7vYi8AeoihNyAJwg4kQdiBJAg7kARhB5Loxhdh0GOnTp0qrb/++ustawcOHCh974svvthRT+eMj4+X1o8ePdqydtddd5W+d9++faX1I0eOlNavuuqq0no2HNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2ftg//79pfWtW7eW1p9//vnS+uTk5Jx7GgQjIyOV3r9jx47S+qZNmyr9/osNR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9j545JFHSuvPPvtsaf3qq68ura9cubJlbfXq1aXvvfnmm0vrixYtKq1XcebMmdL62rVrS+tMJzY3HNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2fvg1ltvLa1v2bKltH7NNdeU1hcuXDjXlgbCM888U1pvN8b/wAMPdLOdi17bI7vtHbbHbR+etuwK2y/Zfr947N2dFwC6Yjan8b+UdNt5yzZJ2hsR10raW7wGMMDahj0iXpX08XmL10naWTzfKenu7rYFoNs6vUC3JCKOF88/krSk1Yq2h203bTe5lxmoT+Wr8RERkqKkvj0iGhHRGBoaqro5AB3qNOwnbC+VpOKxfCpPALXrNOwjkjYUzzdI2tOddgD0iqfOwktWsJ+StFbSYkknJP1E0v9I+o2kv5P0gaR7IuL8i3hf0Gg0otlsVusYF41237U/ffp0aX10dLSb7VwUGo2Gms2mZ6q1vakmIta3KH2rUlcA+orbZYEkCDuQBGEHkiDsQBKEHUiCr7iip95+++2OapL02GOPdbud1DiyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjLOjksnJydL6hg0bWtba/Yns4eHhjnrCzDiyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjLOjkl27dpXWy76zvmdP+XQD8+bN66gnzIwjO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kwTg7Sp09e7a0vm3bttL6Lbfc0rJ25513dtQTOtP2yG57h+1x24enLdts+5jtg8XPHb1tE0BVszmN/6Wk22ZYvi0iVhU/z3W3LQDd1jbsEfGqpI/70AuAHqpyge4+24eK0/xFrVayPWy7abs5MTFRYXMAqug07D+X9DVJqyQdl/RoqxUjYntENCKiMTQ01OHmAFTVUdgj4kREfB4Rk5J+IWlNd9sC0G0dhd320mkvvyPpcKt1AQyGtuPstp+StFbSYtsfSvqJpLW2V0kKSWOSftC7FlGnjRs3ltYPHTpUWj9w4EDLmu2OekJn2oY9ItbPsPiJHvQCoIe4XRZIgrADSRB2IAnCDiRB2IEk+IprcqdPny6tv/zyy6X1yy+/vLS+cuXKOfeE3uDIDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJMM6e3MjISGl9bGystP7CCy+U1hcsWDDXltAjHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnG2S9y7b6vvmXLltL6woULS+s33HDDXFtCTTiyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASjLNf5MbHx0vro6OjpfVGo1Fav/LKK+fcE+rR9shue7ntfbbfs/2u7R8Wy6+w/ZLt94vHRb1vF0CnZnMa/5mkH0fEdZL+UdJG29dJ2iRpb0RcK2lv8RrAgGob9og4HhFvFc8/kTQqaZmkdZJ2FqvtlHR3j3oE0AVzukBne4Wkr0v6vaQlEXG8KH0kaUmL9wzbbtpuTkxMVOkVQAWzDrvtL0v6raQfRcSfp9ciIiTFTO+LiO0R0YiIxtDQUKVmAXRuVmG3/SVNBf1XEbG7WHzC9tKivlRS+WVfALVqO/Rm25KekDQaET+dVhqRtEHS1uJxT086RFuTk5Mta/fff3+l393uK6xnzpwprV9yySWVto/umc04+zckfU/SO7YPFsse0lTIf2P7+5I+kHRPTzoE0BVtwx4R+yW5Rflb3W0HQK9wuyyQBGEHkiDsQBKEHUiCsANJ8BXXi8Brr73WsrZ79+6WNUm69957S+uPPvpoaZ1x9AsHR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9gvA1B8Cam3z5s0ta8uXLy9978MPP1xav/TSS0vruHBwZAeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJBhnvwC0m3b5lVdeaVl7/PHHS9+7bNmyTlrCBYgjO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kMZv52ZdL2iVpiaSQtD0ifmZ7s6R/kTRRrPpQRDzXq0Yze/DBB0vr8+fPb1m78cYbu90OLlCzuanmM0k/joi3bC+Q9Kbtl4ratoj4j961B6BbZjM/+3FJx4vnn9gelcRtV8AFZk6f2W2vkPR1Sb8vFt1n+5DtHbYXtXjPsO2m7ebExMRMqwDog1mH3faXJf1W0o8i4s+Sfi7pa5JWaerIP+OkYBGxPSIaEdEYGhqq3jGAjswq7La/pKmg/yoidktSRJyIiM8jYlLSLySt6V2bAKpqG3bblvSEpNGI+Om05UunrfYdSYe73x6AbpnN1fhvSPqepHdsHyyWPSRpve1VmhqOG5P0gx70B0mffvppaX39+vUta9dff32328EFajZX4/dL8gwlxtSBCwh30AFJEHYgCcIOJEHYgSQIO5AEYQeS4E9JXwCefPLJulvARYAjO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4k4Yjo38bsCUkfTFu0WNLJvjUwN4Pa26D2JdFbp7rZ299HxIx//62vYf/Cxu1mRDRqa6DEoPY2qH1J9NapfvXGaTyQBGEHkqg77Ntr3n6ZQe1tUPuS6K1Tfemt1s/sAPqn7iM7gD4h7EAStYTd9m22/9f2Edub6uihFdtjtt+xfdB2s+Zedtget3142rIrbL9k+/3iccY59mrqbbPtY8W+O2j7jpp6W257n+33bL9r+4fF8lr3XUlffdlvff/MbnuepD9IulXSh5LekLQ+It7rayMt2B6T1IiI2m/AsP1NSack7YqIfyiW/bukjyNia/Ef5aKI+NcB6W2zpFN1T+NdzFa0dPo045LulvTPqnHflfR1j/qw3+o4sq+RdCQijkbEWUm/lrSuhj4GXkS8Kunj8xavk7SzeL5TU/9Y+q5FbwMhIo5HxFvF808knZtmvNZ9V9JXX9QR9mWS/jTt9YcarPneQ9LvbL9pe7juZmawJCKOF88/krSkzmZm0HYa7346b5rxgdl3nUx/XhUX6L7opohYLel2SRuL09WBFFOfwQZp7HRW03j3ywzTjP9Fnfuu0+nPq6oj7MckLZ/2+ivFsoEQEceKx3FJT2vwpqI+cW4G3eJxvOZ+/mKQpvGeaZpxDcC+q3P68zrC/oaka21/1fZ8Sd+VNFJDH19g+7LiwolsXybp2xq8qahHJG0onm+QtKfGXv7KoEzj3WqacdW872qf/jwi+v4j6Q5NXZH/o6R/q6OHFn1dI+nt4ufdunuT9JSmTuv+T1PXNr4v6UpJeyW9L+llSVcMUG//LekdSYc0FaylNfV2k6ZO0Q9JOlj83FH3vivpqy/7jdtlgSS4QAckQdiBJAg7kARhB5Ig7EAShB1IgrADSfw/Z9T2BEXaBJ4AAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plt.imshow(X_train[25,].reshape((28,28)),cmap=plt.cm.binary)\n",
    "# plt.imshow(X_train[11,].reshape((28,28)),cmap=plt.cm.binary)\n",
    "plt.imshow(X_train[18,].reshape((28,28)),cmap=plt.cm.binary)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eab871b5",
   "metadata": {
    "papermill": {
     "duration": 0.033301,
     "end_time": "2022-04-19T13:17:09.529981",
     "exception": false,
     "start_time": "2022-04-19T13:17:09.496680",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Plot a single Augmented Images by taking a single row of of data from the 42,000 number of samples\n",
    "\n",
    "X_train[25,] => Will return all the columns but only row-index number 9\n",
    "\n",
    "But to plot with the use of `datagen.flow` - I have to convert the above randomly selected row to a 4-D Tensor of shape `(1,28,28,1)`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "07a76ef9",
   "metadata": {
    "_kg_hide-input": true,
    "_uuid": "fcf6daaae4424b95978856d7e75271c97b971c71",
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:09.622271Z",
     "iopub.status.busy": "2022-04-19T13:17:09.615280Z",
     "iopub.status.idle": "2022-04-19T13:17:10.831680Z",
     "shell.execute_reply": "2022-04-19T13:17:10.831180Z",
     "shell.execute_reply.started": "2022-04-19T13:08:54.550048Z"
    },
    "papermill": {
     "duration": 1.26905,
     "end_time": "2022-04-19T13:17:10.831830",
     "exception": false,
     "start_time": "2022-04-19T13:17:09.562780",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA04AAAEDCAYAAAAC+VQaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAABUYElEQVR4nO2dd5xU1fnGX2JFBJQqIFWqKEWlF0EFFIlgQ0RQsSFRkhgRjSWWYBJjI4pdsaCAiCgK2KWI0qVIR3oHQZqigvr7Ix/ffM/97XCB3Z2dmX2+fz3LzNy9e+bcc+7lPOd5C/z6668mhBBCCCGEECIxv8vrExBCCCGEEEKIVEcPTkIIIYQQQggRgx6chBBCCCGEECIGPTgJIYQQQgghRAx6cBJCCCGEEEKIGPTgJIQQQgghhBAxHBrzurLKhRBCCCGEEPmJAln9o1achBBCCCGEECIGPTgJIYQQQgghRAx6cBJCCCGEEEKIGPTgJIQQQgghhBAx6MFJCCGEEEIIIWLQg5MQQgghhBBCxKAHJyGEEEIIIYSIQQ9OQgghhBBCCBFDXAFckQGsWrUq+HnChAmuf/nlF9clS5Z0fdRRR7k+5phjXB977LFZ/ruZWZEiRbJ7qkIIIUS22bVrl+tff/3VdaFChVz/7nf5+/+Ov/vuu+DnI444wvWhh+r2UIisyN+jhhBCCCGEEELsB3pwEkIIIYQQQogY9OAkhBBCCCGEEDHIxJoP+Pbbb4Of33//fdeffPKJa3qat2/f7nrv3r2uuSeK2iz0kRcoUMA190LVrVvXde3atV137drVdZ06dYLjHnnkkZZs5s+f7/qnn35yXapUqeB9JUqUcH344Yfn/okJIUQawv00q1evdr18+XLXK1eudM29uStWrMjy36M/79mzx3XlypVdt2vXzvW5557r+qSTTnKdF/PM/rJly5bg5yVLlrj++uuvs/x36rVr17rmPNWsWbPguB07dnRds2ZN1wULFjyY0xYiI9GKkxBCCCGEEELEoAcnIYQQQgghhIihAO1VWbDPF0V6EI0cnTt3rutp06a53rZtm2tGue7YscP1zp07Xf/444/BcU888UTXtAYw8nX06NFZ/o4aNWq4vvvuu4PjtmrVyjVj0nOTe++91/WTTz7p+pBDDgnet3nzZte0NDLavXTp0ln++3HHHZelNgttjGzXevXquZZ9QojsM2LECNevvvpq8Botx7y+OQ7Q+kR92GGHZfnvZmFZh/Lly7tu2rSpa173PFYqs2zZMtcXXXRR8NrChQtdFy1a1PUPP/zgOlGblSlTxnW1atWC4/J9nHc+//xz17zP6dmzp+tbbrnFdbly5f7f35NbcA5++eWXXc+bN8/1ggULXNOqaBbaChkhznmUbVm4cGHXU6ZMcc052MysQ4cOrh966CHXVapUSfCX5C20KfLeglsN2Nd4DZuFbbc/mn2N/872jf5Ozvm09qfaNf3999+75vaEaNmZfEaBrP5RK05CCCGEEEIIEYMenIQQQgghhBAihpRK1eNy+qZNm4LX1q9f73rjxo2xesOGDVn+exQusV522WWuGzVq5DrVllQPFFZKNwv/NuoDJWrzZKIRl30nTZrkevjw4a65nD5nzhzX33zzTXDcn3/++aDP8WBhX+Dfwn5lZrZ161bXTD6iBYK2Hlry+HcyWcostA/RmnL99de7Puecc1wny8KYmyRKbKRmX9if9+zrfbSQ8jvi9cKkSZGZnHDCCa6j3zctVbT5cOxjf6OFqmrVqq53794dHJdjBe2+tOj27dvXdYsWLVyn8nx0/PHHu77vvvuC11555RXXtCyVLVvWNS151BUqVMjyd5iF886///1v1xMnTnTN8ZE2yajFKlmwL/HehlZHWrFp0TYL24Oatnb+nevWrXPN+ThqXaMVP2pLT0U4N8+aNcv1Bx984Jr2Om47MAuv40TbVmi74/zBtqNtz8ysWLFirlu3bu26e/furnmtp8I1zYTlW2+91TVTmStVquQ62j94TfPvT2RVpC5evLjr6LYF/sz3Re9rk4lWnIQQQgghhBAiBj04CSGEEEIIIUQMeZKqR1tSly5dXNOudfTRRwefodWBdgouFzK9jIVKaZ/gkrVZWHCPx+KSP4uzZoIlan/gkjSXwxNZ08zMFi9e7PrNN990/dFHH7lmch8Tq6688krX/fr1C47LpVraq9IFphqyLVm4cejQocFnnnrqKddMuLnwwgtd33///a5TNfUoCm0iTLcyM1u0aJFrLsNHCzhnBfvFvmwPfI22AxZ7vOmmm1yfdtpprnOzwPGaNWtcP/LII66ZEsaxmjZXWpWiP1OzH9FmQusT26RPnz7BcdOljx0obNeopY6Jb4nalW1J2x77C+1YZuH1PWTIkCzP69FHH3Xdo0cP16lcqDW3YLsyec7M7J///KfrMWPGuOaYwKK3t912m2sWwE0m7FdMG+T8yLRF2vHMEo9x7L9ffPGF6yeeeML1qFGjXHMONjN7+OGHXd94440Jzz9VoLVw9uzZridPnuw60TVsFo6J1HwfLbd8D+34H374YXDcmTNnuub4wgLD99xzj+tatWpZXsPkxpdeesk1t1fwOmS7moXjYKIxkbZHtjH7bdQCyLRLWlFvuOEG15y3chil6gkhhBBCCCHEwaAHJyGEEEIIIYSIIU8io5iiwwJ0AwcOdM0lPbMwTSOR5nIdl/e4nPvMM88Ex6W9jDB9JZo8k0nQksf0QaZJcQl8/vz5rqOWCbYll16ZDnTyySe7vvbaa11fddVVrvMyLSU7JFqepmWE7Ud73rhx44Jjsf1o02jYsKHrdCxMxwKNvXv3Dl57/vnnXSdKv6O1hLpIkSKuTz311OC4tFTQXvDGG2+4Zl9mgcjXX3/ddZMmTf7f35NT0H7MfsQC1fx72b+idhvCRMZExVmnT5/umlbKr776KjgW7WV5ZXHKDdh+USv2gVqzaT/hd/fss88G72PqF+0/tKIwSS+a2pWpsKg6rfu0CzGdzyxMJ2USH4u5Xn311a5Toe/SbhlNzNsfeP1z3mbRX7bTZ5995pp9+g9/+ENwXNqf0gGmIjZv3jxLnV14fXJ7AsdNtq9ZOH8lKpa9r3E7L6A1lP2A9kLa5mkrjf6cSHMLDedlvodzkJnZjBkzXLPg8erVq10zwbN69eqW22jFSQghhBBCCCFi0IOTEEIIIYQQQsSQJ1Y92g4uuuiiLPXBwCVV2kyGDRvmeuzYscFnaA1gQdEzzzzTNW1Amcann37qmkVXaVdaunSpa9qIWMDMLEzdokXo9NNPd83vmIXR0gWm4pmFaWhsJy4vs8/RfpKoSK6ZWdOmTV1369bN9e9//3vXLDKXLvCcadXM6ueDJVoAl/aCkSNHuo4mGf4Gv4tkpTjScsx2YD/gedHaFy3ayp95jdKex0KbLBxJy0TUikvLc36ENjymi44fP941rZ0TJkzI8v1moX25c+fOrm+//XbXTNpKNVtPdqGVifabwYMHu2ZbMok32hYNGjRwfckll7i+/PLLXbMYbDrC9jIzW7JkiWta8t555x3XTG2tWLGia6aZMjHYLPP6WRy8pjm+cc5g8jLnj/fee88129osnDdatmzp+rLLLnPNJNdUgPML781y8j6N7c20YSZL8j7KLJzPeb/OMZXHSgZacRJCCCGEEEKIGPTgJIQQQgghhBAx5IlVL7vQLsWimZ988onrd9991zWLkUULoLEw3l133eW6bt26OXOyKQgLsjE9j5Yy2idoz2OSW7ToWOvWrV0zHYopJ+loe2R6C4sHmoXFfVlwkMvItE7RpkqLCQtBm4WpQLRARgtD5zfYd7mEz6X6aBockzT5/TG5kMWzWfixUaNG2Tzj/YM2OiZPUh8MtINOnDjRNS09URvZb5x99tnBz6lQpDHZ0KYzevRo1+xTtN8yLY3XKsdDM7NevXq5bt++vev8UmB96tSprgcMGOCa4+mmTZtccz5u3LhxcCza6pmqx7Q9jrvp0sZMAGXfMzO79957XXMOZ7F4tsvFF1/s+qyzznKd7hbG/YHtyD5lFt7nfPnll645l7Ovbt682TXteGx3s/Cavuaaa1yfcsopB3TuqQznYrOwnVmgnXMz2++tt95yzf4dtT3SnsdETLYxrajJQCtOQgghhBBCCBGDHpyEEEIIIYQQIgY9OAkhhBBCCCFEDCm7xykav8n4XPrzuWeBkYaMk2RFZEaUmoWVs9Mx3vlgYOQo48HLlSvnmvvF1q5d65q+8ejeCPrTGXFcuXJl19y7c+qpp2b5u6PxynkNo18HDRoUvMbYdvZZep4ZR8q43DZt2rimB98s3POS3yJio7DMwPr1610zBpp7GhkDbRb22UKFCrmuU6eO6969e7u+8sors3fCSYB9jdXszczmz5/vml59Xp/c90mvOseDvn37Bsfl/sZMJerbf/LJJ13379/fNfsk9zU1adLENUsHRPeLHX/88a6j136mwrblPgbG3jMOn3Buj8bic58z9+wULlzYNaOWuRe3Y8eOrlNt/y1jxvv16xe8xn237H/cD8t9H9xzMnnyZNccA83Cfsl5J1ouI9XhmDh9+nTXH3zwQfA+7q1ZvHixa5YL4PXJ9uW13qNHj+C4p512WpbHSkc41/CeL7oXiffoCxYscM32534xzsuEc7RZmEPAvXq8jpN9764VJyGEEEIIIYSIQQ9OQgghhBBCCBFDgag1IcI+X8xNGJ1rZvbCCy+45nIrrTv169d3zcjNq666yvUJJ5wQHJd/f363RzFOcsOGDa537Njhmku1a9asCT7P99FOwSVcWvi4JMsIaFpczMLvJS+gVfH9998PXuPPK1ascM1laFoPy5Yt65px7uedd15w3NNPP9118eLFXWdyv6QlgNc1Y0ufffZZ14wdZx+JWiNor+jQoYPrG264wXU0TjYV4TU1e/Zs1ywjYBba89hGtOuw79GuxH7YsGHD7J1wGhKdD4cOHer67bffdk07JG171Iy453VvFsZrN2vWzDXj50uVKuU60657zi+MhKbtjmMAv5eoVY/jY4kSJbL8fa+//rprXhO0i0ftcLSxMXo6N+Hf+fe//901LaNmZhs3bnTNeZT2RPY5xrRznGVpF7OwdAjtZ7Vr13adapbGrKBdm+3ILQhmYXtznma/4PVJq9iJJ57oOlomJFn9JRkwpp335bTKm4XbFtatW+c6kc2T1ypLs5x//vnB+xhBXq1aNddJso5nOfBmzrcrhBBCCCGEELmEHpyEEEIIIYQQIoaUsurxXO6///7gtSeeeMI1l6n5GS5Zc0mPNpyqVasGx23atKlrpu/R0sdlbnHg0A7w0ksvuf7nP//pmsvkTDgzM7vppptc57VlZffu3cHPS5cudc1+SRsjbT38PJORaGE0C5f+77jjDte0CqRb0lEcy5cvd33zzTe7plWPJEozY9qOmVmnTp1cM2mKn09Vmy7tde+9955rWo+ilomff/7ZNf9e2k9atWrlum3btq45VqZSO6QCvL55vbKyPa97JsTxOzELbb2TJk1yzdRRJr7Wq1fvYE45LaB1jDq76ap79uxx/fnnn7v+17/+5Zq2/2iKJOedvLDyMkmQY6NZ2P/YTpxr9+7d67pGjRquObfQGmkW3nfR6nfrrbe6vvTSS11zDM1reC84Y8YM19zm8dlnnwWfoR2U1yjbjveFTB2sUKGC63POOSc4LucjWnZTLTE4EWxL3qexLaN9kp/h39y4cWPXTBvkvTf7GhOWzcL7+jywQMqqJ4QQQgghhBAHgx6chBBCCCGEECKGlF03pH3ELCxwxVQcLu3T1rJy5UrXTIFh2oeZ2cCBA10z3YfpUrT6RG1AIp6jjjrKNYvBMvlmX9YjpuzRgpkXRBPbmPhCfaAwuc8sTDFi+huL8d51110H/ftSHVpAmC7G5EYybdo019HCerRCMU2OBYiZZkZrbrQYX7Jh+hjHKqZDRYuFlyxZ0jVtD0yBopXiu+++c03bWSrZcFIBtlm7du2ydSy2+X333ef6qaeecj137lzXY8aMcZ1phYhpv8lJKw7tt7SsRhMOf4Njq1k41uSFVa9mzZqumQxqFs6j2RmjaD81M9u1a5drpgwOGTLENVPmmMKX19BaTEsY25HXlFk4b9BGTws+LZO8l2GR12jRdRbN7dKli+vLLrvMdeXKlRP9KSkF+z6/72giIxOX2a+YtsfP8H59XwmWtESmypykFSchhBBCCCGEiEEPTkIIIYQQQggRQ0pZ9bjUGi2+SCsN04qYUkIbVbQgWSI2b97sunv37q5ZcI4FwAYPHuyaaSti/6BVkjYkEl2q5ZJuphJNbmT/4/L+K6+84rpr166uo4Wd0xFaF5i+SGsui70yBYkJP6tWrQqOy0J7TDO75557XNOW86c//ck125iWwWTBMY1WUBYB3bJlS/AZJkKxXUaPHu168uTJrpk0yn7E1KPo7xfZg/YqFmXm9U2LKS1FtJiK/YPWSF4TtP5E07wSWfrygsMPP3yfPx8stJ+aJZ5HuPWBlqx0gPeCTHjL6uc4OOew2DXnFTOzq666yvV//vMf13PmzHH9xz/+0TXTNFMB3ovzb+HY89FHHwWf4bzLxGBeb7RDzps3zzVtshzrzMJ5h2mwvF+gLTcZaMVJCCGEEEIIIWLQg5MQQgghhBBCxJBSVr19QctKNNksOzCBihadcePGuZ44caJrLiPKqpcYLmnT6jh06FDXLL7H5BQmBpmFxfsylWhh3Wj642+wXZn8kwlWPUIrCgu3Uh8MLNK3bNky12eccYZrFt9lWhJtVGb7bwfODrwuWJySlg+mO5mFyVjsR7TAMomPRSDfeOMN17RFmJldeeWVrqPXqDh4OA7+8MMPWb4nVdKk0gna8B9//HHXtKkyJbdbt27B55NxfacatEUTjpu09uY3WHSe96HRexQWU2YBe6ZjMkmOyX+0lqcavOe9/PLLg9eYmEfrO6/Db7/91jXt5rT2TZ8+PTgu0x2ZfN2jRw/XTPvjd5RbaMVJCCGEEEIIIWLQg5MQQgghhBBCxJA2Vr1kQFsLl2FZFDIvCuGlKmwXs9DG+PHHH7um3WnmzJmumZTIInEseGtmVrhw4eyfbArCpW0WiTMzGzlypGsmR9KSx/YT+wfTgpjK079/f9cXXHCB67feest19DuilS0Z9gCOSY0aNcpSHwy0T/Tq1cs1i16ahdZGtsuxxx6brd+fH6DF1iwsYMwESb6PKa/169fPvZNLc5huxna99957XfPaZRF1Fh9m0dRMhjZdtpdZmJRWtGhR13369HGdylayvCKaBFykSBHXnBuY7sikQtrZ0qV9eV9iFv7N1Mcff3zssXgN16tXL3iN94xMKKRtjynEybhH14qTEEIIIYQQQsSgBychhBBCCCGEiEEPTkIIIYQQQggRQ77b48RYTbOwevHo0aOz/Myll17qukqVKrlzYjkA/zbqvXv3Bu/bs2dPlp+nZ5WfZzzkqFGjXM+ePTv4/Nq1a13Tv8to6bZt27q+9tprXbOCdybEwLL9GM3JGGnG4o4YMSL4/OLFi1136tTJNauNa79d9uB+J0aj8t/5PbIyulm4VyAZe5xyC+5R6tmzp+vo/ofx48e7njVrlmtGOud3ONbOnz/fNfcxmZm99957rrl37NZbb3XN8TETxsTssGXLFtcsD2IW7gf99NNPXfN6rVu3rmvuk8hueYN0Ydu2ba655/jpp58O3sd9IyzLcNlll+XeyWUgkyZNcv3NN99k+R7u/SlWrFiun1Mqw3vEaGkV9j2Oj7wX5X0R57MjjjgiR8/zN7TiJIQQQgghhBAx6MFJCCGEEEIIIWLIWKseLTbbt293PWPGjOB9L7/8smtaUWgpYxRnyZIlc/Q8DwbGB9MSt3z5cteMvaSlyCxctucyMi15XGpOVM0+anssU6aMa8aLN2/e3HWTJk1cV6pUyXU02jJZJLI3Mh6TNqzDDjss+Dzb5ssvv3RNywjtDwsXLnS9aNEi19FI5yuuuML1jTfe6DqVraLpAL/jrVu3un777bdd8/umhaJZs2bBsfKqz+Ymu3btck3LYhRapzKJaGw4YeQwx9f333/f9aBBg1xzrom21zHHHOP6/vvvd33VVVe5zmT7DucktuXGjRtd0w7Kufmzzz4LjsXxlaUraOu57rrrXFesWNH1vvp4usPyKowZHzhwoGtax83CUiBdu3Z1XahQodw4xbSGcz/b1Mzstddec717927XtN2feeaZrosXL54LZ5ie0DZvFo4JhH1y9erVrnmPmVtoxUkIIYQQQgghYtCDkxBCCCGEEELEkPZeE1pvuMTH5ftx48a5fuONN4LP0zrFJb4bbrjB9cknn+w6melZtOQxSYhpVzx/pgitX7/eNe03ZqE9gXYjar7nyCOPdM22OOecc4LjXnTRRa6ZAsVK2FGrW7LgUu6SJUtcMwmwYMGCrtn2tHqyAr2Z2dSpU13T5vP999+7pi2lcuXKri+++GLXrVq1Co571llnuS5VqlT0zxH7IGoh5XexYcMG1//4xz9cM6GHqT5//etfXdNaapY5Nh9aSWiD+vHHH4P3sR8yBTOVoeWW1/SmTZtcM2WU84ZZOI7SLkYrbiJbNNvr6quvDo5L61iNGjVc51YKVF4QvQ7ZzrQ1f/HFF65pF2fqKMfpIkWKBMflWNm3b1/XTZs2PZjTTms4Pz3wwAOueQ901FFHueYcZBZaRaPpZvkJXsfczsB7ryFDhrimrdQsHDeuv/5617Q/1qlTx3Um2r7jSHTvPnPmzOB9HB8IU4WZjpmMMVQrTkIIIYQQQggRgx6chBBCCCGEECKGtFwfpIWESXIsYDt8+HDXtLNFbU89evRwfckll7iuV6+e67yyl7E4Ha0NtC3UqlXLNe1eTHxZunRpcFwWaaSlrly5cq5pW6Q9r2zZsq5pgzELi5jlBbTjMSXNLLTWsP34GaassY34d0XtJ+yLTMarX7++ayYVNWzY0DXTnUqXLh0cNz8u3R8o/I5ol9q8eXPwPvaFxx57zDVte7ST9u/f33WbNm1cp3OR2ygcH1588UXXtJ/Q0mNmdscdd7hmgmYqwL7A65s2D44BtC6tWbPGdTRBlDZlWkCKFi3qulq1aq45bl5zzTWuacvJNDgPcJ5iYq2Z2eDBg13TPktow6tQoYLrCy+80DWvSTOzBg0auI6mk6YbnF9osaf1yyy0hg0dOtQ1rbZMz2Wx1e7du7vu3LlzcNyaNWu6zg9zENMumSTMFEK2NbdDcP5gSqaZ2S233OL68ssvd037Y17dVyYbbmGgXZr6gw8+cB0dN3j/3qhRI9e8d0922rBWnIQQQgghhBAiBj04CSGEEEIIIUQMBaLWowj7fDEn4DIe9dy5c4P3MQmNlrx3333XNe06LB5Ie1S3bt2C49JaQRtaKthyEtnIaCehvY7FeVmscceOHcFxuURMqx4/ky4w8eb22293zSV1szABjd857UhM1GLRNbZR1KLUunVr17Q0MqGPdh8VEkwMr3+mGjGNi/YVJnC9+uqrrseOHRscl3YMfpe1a9d2/fe//911y5YtXafjNUFoYaPl5JFHHnHN8ZRjxeOPPx4ci9Yz9ulUYP78+a7ZF2jPo72O3z1TBVesWBEcl9Yajq+87k866STXmVy0ljCVkNYaWmH5HrMw5Y79h2Mwi6XzO6LVkbZas/S3lNHuvWDBAtdMFWR6q5nZ7NmzXS9evNg1LZBss2uvvdZ1r169XKdqMmi0EDXvUzk38N+5JYH2W7ZpNKGNNjCm51Jz/ilfvrzrtm3buuZ4YBbeczL9Ld3nExJNXaVFnjZGztksCk6L6bx581yzvc3MzjvvPNecg2jRjY4JOUiWF0jmfItCCCGEEEIIkUvowUkIIYQQQgghYkiaVY+WKi6X0lZB6000RYbLgkw3YloM0424dMq0OS7vicyCliwmtpiFy/u0ZNJSR6snLU5MzMmkpfbchOMK25K2KLa3mdmcOXNcr1y50jWTosaMGeOatjJaTqJpcEzIZMJRp06dXKezxSpa4DpRwW8W0V63bp1rJpn95S9/cc0irWapnVi2bNky10x2Y9+rW7eu60TFPTmGmIUprKlg304V/v3vf7umNZbX4T//+c/gMyeeeKJr2nFoi2K6aKaNtZyDaFl68sknXTMBlONjtO/RmnTaaae5Zkpex44dXUfHxFSB1ydthpwLzMJxn8mfnOfZ99h32O7R1EwmQnIcZLog7yXbt2/vmveevKcwy/uE4X3BNmfb8J6biYwcE2m/5fYRs9AeOW3aNNf8LhMlF3MLxJVXXhkc9/zzz3fNMSRJ2x5k1RNCCCGEEEKIg0EPTkIIIYQQQggRQ45b9Wip4xLdoEGDXLO4GJdOuXwftUzQZlG9enXXXMZjkgnT5oQQuQ+tJSwm+Prrr7tmYWraAaLQakGbSqKEQiYadujQIThW7969XadDihTtE0wTpb2OiVsTJkwIjkWbI9ON2I60L7OQIFPNokUdU7XtRPJh4XnawKKFvPM7O3fudD1ixAjX//rXv1wzyZFbCZgAGm1XXqennHKKa25X4OdTCc4TM2fOdP3ss8+65pxhFo6JLHZKuza3g9DKyHGMtm2z8J6RbUcbHu26UUteXkOrIW3wTF0cP3588Blatmkp5PzA+ZdWcPZVWvPMwjmMyc38HWxX2ko5H0WL2fK7zAMLpKx6QgghhBBCCHEw6MFJCCGEEEIIIWLQg5MQQgghhBBCxJC0OHIhhBBCCCGESAO0x0kIIYQQQgghDgY9OAkhhBBCCCFEDHpwEkIIIYQQQogY9OAkhBBCCCGEEDHowUkIIYQQQgghYtCDkxBCCCGEEELEoAcnIYQQQgghhIhBD05CCCGEEEIIEYMenIQQQgghhBAiBj04CSGEEEIIIUQMenASQgghhBBCiBgOzesTECI/8csvv7j+/vvvXa9bty5438qVK11v3LjR9aGH/u+SrVOnjusqVaq4PvLII3PmZEVG8euvv7r+4YcfXK9YsSJLPWfOnODzNWrUcN2yZUvXxYoVy8GzFOK/cKzcuXOn66+//jp43/z5812z/5YsWdJ169atXZ9wwgmuOZ4KcbBs3rw5+Hnp0qWuv/zyS9elS5d23bx58yz/PT+ye/du19u2bQteW7RokesdO3a4/umnn1xXrFjRdbVq1VwXKVLE9e9+l3PrRFpxEkIIIYQQQogY9OAkhBBCCCGEEDFonVqIXGb79u2uaX+aMWOG61mzZgWfmTdvnuvVq1e7PuKII1yfddZZrrt37+6aFgBZUfI3tOetXbvW9bRp01y/9dZbrmfPnu168eLFwbFo1Wvfvr3rO++80/VRRx2VzTMW+ZnvvvvONfvo+PHjXbOPmpktWbLE9Zo1a1zTQrpw4ULXf/jDH1zT1lOgQIGDPW2RT9i1a5dr2kKffPLJ4H2c2znnV69e3XXjxo1d33zzza6rVq3qOiftZanAzz//7HrSpEmux4wZ43rDhg3BZ9jO33zzjetVq1a5rlevnutrrrnGNeepnLSUZ9a3IoQQQgghhBC5gB6chBBCCCGEECIG+XiEyAW4vPz++++7HjFihOvJkye7ZmqUmdkxxxzjulSpUq5p+xs+fLhrLumXK1fONa0oIv/x1Vdfuf7HP/7hevTo0a5pPznuuONcV6pUKTgW7U7r1693XbduXdeXXHJJ9k44xWD6IJMvJ0yY4Jo222+//db1BRdc4PqMM84Ijlu4cOGcPM20hm327rvvun7jjTdcf/bZZ645BpqFFhz2X1p+hgwZ4ppWKL6fCVzpCq25sh7mDHPnznX97LPPun777bdd005vZnb00Ue75vy9bNky17Sase9df/31rpkAmU6wH3LcfPrpp13zmqadkeNBFN7b0Bb++eefu2YKMZM1zzzzzOBY2dnGoBUnIYQQQgghhIhBD05CCCGEEEIIEUPGWvVYHGvPnj2uo4XKuAy7ZcsW17Q+0bJy0kknuT722GNz5FzTCRYlpI3FzGzTpk2uaeWhZYLfBS0TTJvhMneqQ5sT+xKtISNHjnTNZWsu4Tdp0iQ4bv369V3Tbjd27FjXgwcPdk1LFu0A+cWqx375448/Bq+xaB6vfyb80FpQtmxZ17QBpWpCIc/dLLSR9e3b1zUtZSySTKvdOeec4zpqE3nwwQddM3GPKWcXXnih61Rtr30RTXT6+OOPXX/xxRdZaqYVsu8tWLDA9datW4PjXnbZZa4PP/zwbJxx+sDrjTYlWp6ee+451ywiWrRoUdcsZmtmduqpp7pmIXCOwbQFsSDp+eef7zpdrXoc0xKNb7SLs4+yXc3CFLLKlSu7zrR0tzjYR55//nnX77zzjuto0XpCa26DBg1cf/rpp67HjRvnmim6tJqlslUvOu+w77G/DRw40DXt3hwTOR+dfvrpwXE5B7MtWTT3sccey/J3vPfee65r164dHPf444+3gyV/XQ1CCCGEEEIIcRDowUkIIYQQQgghYkg/L0UE2sVos5gyZYrrl19+2XXUMsFlax6LtrOKFSu6vu6661x37tzZdU4W10o12EZMj+GSqFlYpJAF4Gi54PIuLRcdO3Z03aFDh+C4qWRliSY6cemdRfAmTpzoumDBgq4bNWrkul27dq5pkTILLXZcxmba3rZt21zXqVPHdYUKFfb5N2QKLIZHGxpti2ZmK1euzPI1Wi1ot2L/69Wrl2t+d2Z52y9ZKJQJeWZmt912m2umO9IC26lTJ9fXXnutaxYSPOSQQ4Ljso3YptG+m26wH73++uvBa7SA0AJLGyItIHv37nVNCyMT4szMTj75ZNennXbawZx2yhO18rD9mK71zDPPuKbllmMgi31fdNFFwXHZlrwmab3ncfl9cWxNZXj+ZqEVinPNm2++6Zp2Ws4bTNuL2mmXL1/u+qabbnJdvnz5gzjr1IftyvsU9k9a7Zn41qxZM9e8LzQzO/vss11z3OXcXKJECde09tWsWXP//4A8hNezWWiz5TYCWg9pDW3RooVrzq1t2rQJjsvC60zB5P3Pq6++6pr3q7SY5uR8rRUnIYQQQgghhIhBD05CCCGEEEIIEUPKWvWiy/wsEEqbCC06L774ouuZM2e6ZvpGNHGLcBmV9iqmpT388MOumcITtQ+kkr0suzDJbcCAAa6ZVGQWppfRAsGlalpZaF9hyhLTfMxC+1BeFPWjvZMpW2Zhe0ybNs01kxjbtm3r+rzzznPNNChaKczC/k7bI5O6aPFh/+PSdiZAOwVtKbSoMUlu/vz5wecT9UsWx2MxPdq1aEel/cAstBJFbW25AVOyaIu94447gvfRTkubMQsrdu/e3TVTBJkSyX5nFtpUmjZt6ppJo+mSpMe2fOSRR1y/8MILwfto46OFhtbY3//+965pS6Flcs2aNcFxaSPLVKJ/My3zjz76qGum0zZs2ND1pZde6pq25tKlSwfH/f77713TbsX5uUePHq5pLU01iz23C9Bq99BDDwXv41xDex3vm3gPcsQRR7jmuPX1118Hx+UYd8opp7jmd5GMsS63iFoeaWW+7777XLO4PO9feK1fffXVrlu1ahUcl/MJ0+ZoHaP9keNpKqc1c/6lNc/M7MMPP3TNa5KJwbTctmzZ0vWJJ57omhZGs3Cs5r0Yx+Y//elPrplWzKK30eNmB604CSGEEEIIIUQMenASQgghhBBCiBhS1lcRXeb/5JNPXI8YMcI1rVO05HFJlEt30eJ5XKpnQVZaKZiwQivApEmTXDdu3Dg4LgvxpQu00XHZmoUEubQdXfYuXLiwa9p3WLSMVpZhw4a5prWS1kCzMAUpLyyQTMeKLk/zNdpMaIWiVa9cuXKuaRudPn16cFymJrHYJi0aXOq+6qqrXKdrsUIuydN+8tJLL7n+6KOPXNPKwpS5aGE7FtTjdcnrnQlB/fv3d01r5Lvvvhsct3fv3q6TYV9hP6B1h9ekWVg8mTY+Jlfy2uU1+cEHH7geOnRocFyOd7RGpDK0LjEVsF+/fq7Zv6IWcRag5N/M1CzaIWkjp9WR46FZmASXqbBdzcwef/xx17Qj0WbMhEdamTk38Xs0M5s8ebJr2r85j/Tp08c156m8sH5HYRowi6Iy0ZPjm1lovaPNiXZSFvetVauWa9pxb7311uC4tDwnKhRcqFChBH9J6sOxzszsqaeecv3aa6+55j0jrZ2cZ2llpDXPLEzf5XXAe0lCK2kq9MlEfP75567ZV8zC+ZsFvi+++GLX7Ie0yu/Zs8c1x1Cz0HrP+x/eP/L64DzN7zEnSc87LCGEEEIIIYRIInpwEkIIIYQQQogY9OAkhBBCCCGEEDGk1B6nKVOmuI7uJ2BEMKOrGRvOmE3GELIqMf3AZmEl4uLFi7tmhCT3tdADT51O+0roJ+Uejrvvvts1vayMIebfyYhYM7MrrrjCNatqs10ZYc49G/RNR6O5o3upkg33vzCa3izc28K+xTjyRYsWuWaMNPfxcf+IWbiXh3uhGI3Kdj3ssMP2/UekAewP7Iv02nMfI/crMfI0Gg3L2HbuMaMvnXvM+F0wXrtMmTLBcZO9327UqFGup06d6pplFMzMnnnmGdfcH8ioY8YQM3qXe6cYKWsWVn1nu6RyPDEjazmOcz7hOM5xy8ysb9++rrk/lntMuG+C+20YfR8dKznvZCrcL2cW9qcWLVq45v4jjm9LlixxvWnTJtfsr2Zmzz//fJa/g99Ronk7FXj22WddM6adfwv3e5mZnXvuua459rF0B+cH/s2c/6N7sxPNZ+m8r4n3L9G+w/3b/Bvbt2/v+sYbb3TN9uW8/NVXXwXH5b49jrW8l0k0BvC7SrX9Thzf1q9fH7zGfYv8O3mfw/tq7kvi3DRnzpzguNx7xv15bCfuV+Zeq9wife72hRBCCCGEECKP0IOTEEIIIYQQQsSQ51Y9LtG9+uqrrmmliL6PMZBcUmVsJC05tJhEYyN5XNoBduzY4fpvf/uba1qFaAdMZesFo1zNQuvTXXfd5ZoWJS7nc3maS9Csom0WLpcmso5Vr17d9S233OKaEdJRWwstL3kBLRNcKjYLl4UZf0tLH/sYl60ZQ7svaCFgtHaXLl1cM1Y/laG9YdCgQcFrjzzyiGvaG0qXLu2af3OHDh1cM+45ei2y/2zbts017XnUbGOOKazubpZ8ey5jWdmOUd58803XbGNe97SMJiJ63dHGxoj96Jia1/Aafeedd1wzvpY2j0suucQ1xySzMDac1zrtuyNHjnTNuGHaUv7yl78Ex00na/fBwlINZqF9hzYf2nLZr2lxnjdv3n79TtpnaW+rW7eua9r78wreR7Bfcq7o1q2ba9oZzcIxkfc3nLdpU+V4yjIDbGMzszvvvNM17ZTpDPsd497NwmuU9zmMaWebvv32265pp49a9VhOg7+fVlTOZbS5pTL7shFybmU5Fdr7Eo0B7Kv7guMmrwHeM5YtW3a/jpUdMn/0FkIIIYQQQohsogcnIYQQQgghhIghz616XNLk8l506a5GjRquWYGcFp1E9gdarWhXMQstF7NmzXL93nvvuWaaVv/+/Q/od+cVP/30k+toYhurZTPljkv+XPqklYUJZVF7GJO2uAQ+evRo17Sy0LbG9mYaUipAuw6timZmS5cudc2UGC7v8+9p0KCBa/b9aFofbVL8zB/+8AfXZ5xxxv79ASkEE44GDx4cvLZw4ULXJ5xwguvHHnvMNS037H+8/nbu3Bkcl/YKWjX4+2iDY3Xzq6++2nVeJxfS5sH0q3Xr1gXv4/VN6xPHQdo/2W/ZjkyGNDO7/PLLXTds2PBATj2p8O8cM2aM67Vr17qmxZvJeUxmNUtss73qqqtc04pCWzSr1ud138kLeA2bhdcYrWOcgwn7Ii14nNuiVKxY0TW/16j9O69ZvHixa/ZLXtdsl+hYybmaFlIm0j788MOuo2PEb7CPmpmVL1/edab0WaYLRi2fnKe5VYNbGDgGrFy50jXnj331ScJ7Cc4t3FqSytxxxx2uuVXFLNyqQHgvz3sh9lWmO0fv/TkO0IbH9rv55ptdJyMBMrXu9oUQQgghhBAiBdGDkxBCCCGEEELEkOdWPS59btmyxTVtX2bhUuiTTz7pukSJElkelwkftLhEbWu05yVKqqJlhYk8qWbPI2yvESNGBK+xMGHhwoVdX3vtta47duzomjZJWsiYPGhm9sknn7hmIb8VK1a45nfM9uO/R1MA87rAZvPmzV1HCyjTcrJ161bXXJLm38nvhYUPaWMxC20ul156qevo8ni6wWV4WpyiMH2HdisWyWV6Gr+HaOFWWiJpt2I/K1KkiGvafVLJNsoC07R80OZgFto+OA6ed955rtl2tOLS5tCmTZvguNEUzVSFcwfHd37fHLuYjhW1RE2cONH1zJkzXUfTNX+Dcw3bNZVTV3MLWu/NwuQxvla5cmXXHENZ2JVjAO3yZmYlS5Z0TSszkx9TDd5HXHDBBa4//fRT1yxCz74X/TzH1OjcmRW0+bGos1l4P5CJcGw3C+9NVq1a5Zr2SVrC2Vd5jxT9fjgf8dq//vrrXTNhOF2oX7++66gVt1GjRq55z8YC4WxLzsv/+c9/XPNe0Cycz3r37u2aVr1k99vUvfMXQgghhBBCiBRBD05CCCGEEEIIEUOeW/WYeMKlTy5Tm4WpMCyUS2sEU2CY6saiXfuCFg/a81q1auU6XQqV0R5Ge5NZmKLFZVDakrjUzCTCJUuWuGYykFm47E37EJNraPVjgUIuZ+e1NS8K+0U0hSj682+wz7HQG/vutGnTXLN4sJlZjx49XJ999tmumS6V7jC9ziy08jBR6rnnnnOdKOmJ9jza/PYFbQO0sdECkOj7zQvOOuss17RFRFOIOCayv1D37Nkzy9/B4uJdu3YNXkuFwqH7A20bTFjkOMQxjWNd1ILHFEimayWCRTRvu+0217RH5RdYoNIsnOtZTJnjPRPi+B3xfiBqn6XdjFbmvC6cvi9oc+rXr59r2t3Z32bMmBF8npYzFhpmcVsWKGWCGce36667Ljguv6NM5Kabbgp+7tSpk2tuG2E7cNzjPM2xZfXq1cFx+TMtp0yFTce5nPdC0VTlREmrtI8yhZhbFXgvGbU1X3zxxa5pF8/Le3GtOAkhhBBCCCFEDHpwEkIIIYQQQogYCsTY2PbP45YNEhUgmzp1avA+Fi6jFYf2MNpqFi1a5JoFMKNw6ZHFD++55x7XLVq0cM0En2jyXyrBNqpZs2bwGtNMaOth8TvaHLgkzYQU2lj2Be1VLBo8cOBA17RtpVKS2cGyYcMG18OHD3dN2xnbL2qZYPpOouTIdITL9kzQMguvcSY/0h5B+07Tpk1dMz2OhR/NwjGC9oJu3bq5vuaaa1wzsS7dYVLosGHDXPfp0yfL948cOdJ148aNc+/EkkSvXr1c0z7DvkdrSHTsYV9gsXCmwtHuzALf55xzjutUsx+nErwH4bX64IMPun7xxRddR5PgHnjgAdecR1J5fj5Qogm2TH3717/+5XrQoEGuOZ8zbZBWvUyaW/aHaOrg7t27XdMaSbsv74U4z/D+hX3VLCzUyte47SOVU5lzEtpMX375ZdfctsB7xPvuuy/4/IUXXug6UdH7XKRAVv+YP745IYQQQgghhMgGenASQgghhBBCiBjyfC2bS3S0PJx//vnB+zp06OCay6W0O9FGFl3a/o2oFYPFXWmzqFOnjmsWhUwXuIzJVDazML2MiVyJEsvKlCnjmqlR0SQZJh/RJlGlShXXf/vb31zXrl079nenE0zkYiHDZ555xjUL3TLJrXv37sGxMtVCsb8JhVE7zm8wdYvpkEwwi1pIabtgcufll1/uOlrYOFMYO3as60ceecQ1bdH8d6Y+ZQK0yXC8Yurn8ccf7zpqXec1/d5777mmfYdph+xHsuftH7RNDhgwwPWQIUNcc57+4x//GHyeVvRMsudxrIsWBX3ppZdc0zbKOZUFmFlEPVPnlv0h2j84NyQqokp7Hwtqjxo1yjUL25uF969MQM1kex77K7eD0ErLbTN8P6979luzMBEyVcjcb1EIIYQQQgghcgg9OAkhhBBCCCFEDHpwEkIIIYQQQogY0sYQzD0w9DxyXxP3ldCPzqrt9evXD47bu3dv16x8nEn+9H/84x/Bz+vWrXPNeOdElawZWU4YJ2kW7nGij5r7SrgfIBP2NRHuX3rllVdcf/31167Zxzp37uyaleBFCKP1uR+C0e6DBw92zVhZM7N7773XNb3nbPNMut7XrFnj+s0333TNfT308zdo0MB1wYIFc/nskgvHLkZVU5Po/jjGYPP6btKkiWtGPVeoUOHgTzYfwT0hH3/8sWtex9xfxu+L5UHMMmse4X6aOXPmuO7Xr1/wPu5d5B6vP//5z67PPfdc14nmcBEPx4Tnn3/eNb8D7gM3C/eUpeMe+YOB+/Aef/xx12+99ZZrZgxUrlzZNdsrFfc0RdGKkxBCCCGEEELEoAcnIYQQQgghhIghbax6hBXchw0b5vqjjz5yTbvOxRdf7PqWW24JjpWpdh0SXaavXr16lpowlpdLsCNHjnQ9efLk4DNcYr3oootcd+vWzXW6W4EYoWkW2qK4JD1+/HjXtC2yCjb7pQhhO69cudL1/fff75qV28mjjz4a/HzZZZe5Ll68uOtMjYadOXOm6w8++MD1scce65r23caNGyfnxFIUWkEnTpwYvDZ06FDXRx11lOuuXbu6ZumKTIrDzk127tzpun///q5ZaqRdu3auOW9nsvVp3rx5rhnj/O677wbvoz2xV69erlniIt3n2ryEY8KUKVNc836T/bBt27bB58uWLZuLZ5d3cF5ev3598Br7KOPy2VfPPPNM1yw3wLE1HcjMOwchhBBCCCGEyEH04CSEEEIIIYQQMaSNr2DatGmuubRP2953333n+sYbb3TNpeyqVasGxy1QoEBOnmbGsGPHDtds49dff911dKmWiXlM96lSpYrrdG9vprqZmY0YMcI1E6GOPPJI182aNXN9+umnu5atJzGrV692/fTTT7tmWiETeo477jjXPXv2DI6ViZYVWkmWLl0avPbGG2+43rBhg+vatWu7LleunOv83g95TT/77LPBa9OnT3fNRNb27du75rUuEsN0sgEDBrieNWuWa9rKmZ5Xr169XD23ZLNnzx7XTLt87LHHXNMWX6lSpeDztCwzPS8Tx7q8gDZufie0qtWoUcN13759g89n6piwadMm19yaYBZe00zNPOGEE1xz3Ew3ex7RipMQQgghhBBCxKAHJyGEEEIIIYSIIWU9Gps3bw5+vv32211PmDDBNZf2W7du7bpLly6uq1WrlhunmHHQ/jNjxgzXTz31lGsmF0ZtjzfffLNrLs+mOyxKOHfu3OC1d955x/WqVatcH3/88a5ZFJP2Mh43P9qlogmFtAHQpkKrHq0o55xzjmsWKs0PdhVaIcaNGxe8xuLfxYoVc82Cyy1btsy9k0szeA0zhdDMrFSpUq6ZiJmpqVm5CS3fL7zwgmvatzmH//73v3edaQmYixYtcv3www+7Zl+k3Ys2eLMwPS9TbWHJhknMw4cPd718+XLXTBum9Znzuln6b0kgbBcmKdMSbma2ePFi17zn4b149+7dc+MUk05mjUZCCCGEEEIIkQvowUkIIYQQQgghYshzfxCXAZlgxKV8s7AwIZftmzZt6pp2HRa6FPvHsmXLXDMxZezYsa4LFy7s+sQTTww+37x581w8u7yDFrJPP/00eI2WRvL999+75lI/da1atXLqFNOSdevWBT8PGjTI9fPPP++alsaTTjrJdYcOHVync0LPwcBCodF0o2+++cY1i1LPnj3bNdMgL7jgAtf5xfbDPvXMM89k+e9mZieffLJrWpFFPEzRMwuvb6ZmspAoNfsxU+hYUDOdYHoj71VGjRrlmtc1r8uHHnooOFZ+uU6TCQutjxkzxjWL3NetW9f1+eef75qWaLNw20O620xpwaOFnoXWo9BK/uWXX7p+7bXXXHfr1s11utnr0/sbFUIIIYQQQogkoAcnIYQQQgghhIghT6x6TCaZM2eO64EDB7p+++23g89wabpBgwauL7/8ctclSpTIydPMd0yaNMn10KFDXXMZlQXM+H1lMrTqffXVV8FrO3fudF20aFHXjRo1cs02O/XUU12nq+UkO9DCyIRGs8RJRiz+yGKPTJrimJJJiUaJoK2Z120UWqFZ1Lp8+fKu84vthwXSH330Udfsa9G2OOOMM1xHk7PEvmGKnpnZJ598kuX7mK7Jgs1M1s2Ea5r3NMOGDXNNCx//Tlodo4WZr7/+etfpZnNKJRIl5nKe5xhK+yitp7SmmYU2vsMPPzxnTjaPmDdvnmva83bt2hW8jzZbWsSZsNewYUPX6TzvaMVJCCGEEEIIIWLQg5MQQgghhBBCxJAnVr2NGze6ZsrGhx9+6JrpMmZmlStXdt24cWPXzZo1y4UzzGy49MyUOC49b9++3TUTCpkqk1+SzGgBbdeuXfAaC2GWLl3a9Zlnnumatr38aM8jTNhhspuZ2YIFC7L8DPsrbQNMe8xvyXBsh927dyd8H4sE33XXXa6jiZj5gYULF7qmbYz2m7PPPjv4TMeOHV3n94LVB0o0cZTWM7Zf/fr1XdOKe9ppp2X5/nRl7dq1rpnURgtokyZNXLNIdZs2bYJjyZ6XM/A+k2PqihUrsnw/7c6katWqwc/pbs8jvMeh1S7aB6tVq+b6iiuucN2iRQvXvI7T2X6rFSchhBBCCCGEiEEPTkIIIYQQQggRgx6chBBCCCGEECKGAozxzYJ9viiEEEIIIYQQGUaWG7G04iSEEEIIIYQQMejBSQghhBBCCCFi0IOTEEIIIYQQQsSgBychhBBCCCGEiEEPTkIIIYQQQggRgx6chBBCCCGEECIGPTgJIYQQQgghRAx6cBJCCCGEEEKIGA7N6xMQIhVhYejvv//e9Y4dO4L38eeff/45y2P98ssvrgsVKpTle0qVKhX8nOh9Qggh8hecQ8zMtm3b5nrZsmWu582b53rFihWut2/f7nrNmjWuCxYsGBz3hx9+cN2+fXvXXbp0cX3EEUccwJkLkXloxUkIIYQQQgghYtCDkxBCCCGEEELEoAcnIYQQQgghhIhBe5xEvuann35yvXXrVtebNm1yvXz5ctezZs0KPj958mTXGzZscP3dd9+5Xr16teuaNWu63rJli+tHHnkkOG7Hjh1dH3bYYfv+I4QQQqQ9e/bscc15Y+bMmcH75syZ43r+/PmuFyxY4Hr9+vVZHnfnzp2uDz/88OC4BQoUcM19vg0aNHB94oknxvwVQmQ2WnESQgghhBBCiBj04CSEEEIIIYQQMciql0EwDnvXrl2uGadtFkabHn300a6LFi2ai2eXmjCy9cUXX3Q9dOhQ14xvpZUh+jOtDXv37s3y36NWv9/49NNPg5+bNm3qumzZsolOX4j/B+2nmzdvdr1u3TrXjDBmH547d67rK664Ijhu5cqVXf/ud/nv/9xorV2yZEmWumTJkq5bt27tWhHOYn/49ttvXQ8ePNj1Cy+8ELyPNrxjjz3W9fHHH++6RYsWrgsXLuya9rxov5w+fbprWgVpXZdVT+R38t/sJ4QQQgghhBAHiB6chBBCCCGEECIGWfXSkJUrV7r+4osvXC9dutQ1k3No14m+j9YAJu/QllK9enXXFSpUcN2oUaPguD179nR95JFHxvwVqQFT8YYPH+6af/9JJ53kum7dusHnGzdu7Jo2iUMP/d+lRdvejz/+6Pruu+92zUrwZqHtUgiz0IK3ePFi1+PGjQveN3LkSNdr1651zWudCZDHHXdclu+hxdTM7MYbb3RdunTpAzn1lIDjG9uF9qSoZZYpZbTZsm2OOuoo17Vr13ZdpEgR17Teiv2DyaQcN2lDNwvnmhIlSrhORzspr78JEya4pqXezOzcc891zTmpTp06rqtWreqafZTtEp13eI1s377ddbly5fbr/NMZXtPse7wX+OGHH1zT8liqVKngWIUKFcqNUxQpQvqNLEIIIYQQQgiRZPTgJIQQQgghhBAxyKqXhjz//POuhw0b5prJN1wqZqKOmVnBggVdM2GPS/O0+n3++eeuJ02a5DpqmWjZsqXr+vXrx/wVqUGlSpVc//3vf3dNeyKX4dl2ZuFyPe15hBaA2bNnu+Z31K5du+AzxYoVizt1kaHQGvLJJ5+4HjJkiOtEhZfNwoLJtNSVKVPGNe097GtTpkxxHS26yd+TLlY9Wms4jr399tuuJ06c6HrRokXB53fv3u2a4+gJJ5zgmmmkX331lWsWKc1kqx7HN84bLMxqFlodaUPjZ5j8yO+iWrVqrnfs2BEc98ILL3Tdp08f11H7VDpQvHhx15dffrnrhg0bBu+jLZw2vAOFNlWz8B6iVq1aritWrHjQvyOViFoT+ffPmDHDdaJ7ISYNMpE3Cu357If8Hjkep+M2h/yMVpyEEEIIIYQQIgY9OAkhhBBCCCFEDGlp1aM1IFqQND9Aa8hf/vIX17R+0VIWLXLHFC0u89O2x3QdWvLuuece17SxmIVJNOlCgwYNXHN5ncvltOBlt799/PHHrmkjihYVzI79Il1gHzML+xntarRXsJgzE6j4vbBdzcxq1KjhmgmJhxxyyEGcde7DJLfnnnvONe15tH906dIl+HybNm1cn3zyya4TFcFkOz744INZ/g6z9CnEzCK+/fr1c820QbZFzZo1Xf/1r38NjnXqqae6Znvw+mQx4e7du7vONHsex/uFCxe65pg2depU1ywcbhZaPb/55hvXHHcT6USFw83C64K/v0OHDgk/k6owFbBbt2658jvYrvwezcK+fPbZZ7vmeJHORK2J//nPf7J8jQmmiRIJOV9t3bo1OC7nIxbIpv2Z2xnOPPNM15yjMgEmYjLNlLZc3oelQ7FwrTgJIYQQQgghRAx6cBJCCCGEEEKIGFLWqsclPTOz5cuXu+ZyKZf7mGjEoq+0mDC1xiy0aVSpUsV1KqfIsBAloaUsUcLbwfDhhx+65rLrNddcE7yPxffShWhKXm7AvvjZZ5+5Zt+jfdIscy2otHN+/fXXwWtM2mJRV6Zr0apH688xxxyT8HfS8nLzzTe7TtVkOPbJZs2aue7Vq5drWsiyO1ax3ZmqRYuj2f8fO1MV9h0WCKcdpG3btq6bN2/uOmqZZeJgojF19OjRWb6/ZMmSB3DWqQ/7xuDBg12/8cYbrmmrjVo7aRtlqiPfx0KvvA5oFWOhcrMwvZAJaCJraJOMJmfSKtikSZOknVOyoE3RLOyH7J+0i7GQNdMMmYoXLVjPa4Xz1IgRI1zT+sr73VSw6vHvYX9h4XUmLJuZrVq1yvWXX36ZpT7ttNNcr1y50jUTYzkepypacRJCCCGEEEKIGPTgJIQQQgghhBAxpJRVjzawxx9/PHiNdgAuCUaXXn+DS/tMz4ombtGuw+XJAQMGuL7gggtc56QF7mBJRoE0JsYwaYtWKy67moVL2uJ/MM2LSVMscJguNqj9hTaEDz74wHX//v1djx8/PuHnef3SssO+TysU3x9NOGIiF8eOVLXq0cLJc0z09x4MTI2iZYK2DBYdNQst0qkMbYxPPPGEa45XbMuDGdM5j9DuRLtyNHU03eFcyyK0TASjvbNq1arB58uVK+c6kT2P3wX7G/todJ6hxSpaSFz8FyYRM3Vy1KhRwfs6duzomja2TOGUU04Jfq5du7ZrphUzoXh/0lejVj2Or0wupI2Yc3758uVjf0cymTdvnmv2ERYLj9o8Od7xPpHjBosG0544ZswY140aNQqOy4LuqUJ6zIRCCCGEEEIIkYfowUkIIYQQQgghYtCDkxBCCCGEEELEkPcbdsCWLVtc0yNqZtayZUvXjGqmL5WefFaGZxzk6tWrg+PSc/nee++5ZjXyTp067c/ppz306Q4aNMj1tGnTXD/88MOuGd8uQhifP2zYMNfcA8DYTVYnzwS2b9/ueuDAga6nTJniukKFCsFnuD+kadOmrhs2bOia1zX3QOzZs8f1c889FxyXXup0iHmnv546J+Fepo8//tg147QZ2Z1OsL/sz/6Eg4ER+Zy3+LujJQbSnVKlSrnu0qWLa8673LeRk6UeFixY4HrGjBnBa/Xr13d97LHH5tjvzCR4vXPPaXSvZOfOnV2nw1h5oLAP5yTRcYZ9f+zYsa55/9m1a1fX0f2kyWL+/PmuuZedsemcZ7kXK7qfkKUcOLczwv2XX35xzXI2nNejOQTa4ySEEEIIIYQQaYgenIQQQgghhBAihpSy6tEm8qc//Sl4jRGkXNZLFCVL6w6X/qLLgKw6ziVs2iyyG/2byjBC8pNPPnH9wAMPuD7vvPNc9+jRw3W6xBMnC0a+vv/++67Xr1/v+o477nCdqnHYOQFjw2lfuuyyy1xXr149+AwrsTOimMdin6NetmyZ66htgvHUtBPkN2jFpfXpo48+ct23b1/XuWVryW1yy57H9uO8sXHjRteMcM40+y3n3datW+f676Pdd/Hixa6jNn7aejPRXpYT0KrHeb5Zs2bB+zhWioOHljza3mh141iRV3a0Dz/80DXvWVgugNc6r7VotDvvZzj2cf7+7LPPXHO+571AOoybuvMVQgghhBBCiBj04CSEEEIIIYQQMaSUVY9LehUrVszWsbj0SU27gZnZ66+/7rpmzZque/bsma3fn8rQUsZ0KKaRMeXlzjvvdC17XmKWLl3q+p133nFNexitETmZOpVq8DqjPa9EiRKuo0vy2bFYffXVV66ZiGlmdsUVVyT8nfkJ2pg47tEWeckllyT1nNKJXbt2uWY6JPt6ixYtknpOmQzT88aPH+86muYaTfcS/4VWxyeeeMI1UyA5NpvJ6niw8J7KzOz+++93Tdse76Vo28srateu7XrAgAGuGzVq5JoWvOxaCqdOneqa20Q49+eW1Ton0V2wEEIIIYQQQsSgBychhBBCCCGEiCGlrHq5BdOQmOphFiZKXX311a5ZWDfTmD59uutHHnnENZfwb7nlFtfR9DPxP7hE/8orr7hmGz/22GOuS5YsmZwTy2O43F65cuVc+R07d+50vWTJEte0npkpKeo3vvjiC9eJxj0VEE3Mjz/+6Prdd991feWVV7rO5KTMZMCEwokTJ7revHmzaxbfNcs/Y+qB8v3337tmkh6TiPNzymh24X3lW2+9FbxGK/T111/vmrbSVLBFNm/e3DX7RU6m/H377beueR3Ttl+jRo0c+33JQCtOQgghhBBCCBGDHpyEEEIIIYQQIoaMterRQjV37lzXTDUxC9NDunfvnvsnlgdECwZ+/vnnrmld7NChg+uWLVvm/ollAEzaoj3viCOOcM1laP67yB4siknrVJ06dYL3sQBvfuOXX35xPXjwYNe05bKQYaKC4iK03NJ+0qRJE9d79+51rbbcPzhXL1y40DVT9apWreq6bdu2yTmxNITX+7x581yvW7fONYuwH3PMMUk5r0xk5cqVrp966qngtQYNGrju1auX61Sw55FkJPt+/fXXrkePHu2a1zETrdMBrTgJIYQQQgghRAx6cBJCCCGEEEKIGDLWS8BipP3793cdtUrdcMMNrrNbdDeVoP1h0qRJwWu0NZ111lmumf6iJfz9g227YMEC17fffrtrphKm2lJ9urFjxw7XtJny388999zgM4UKFcr9E0tRmGK0atUq10xTqlChgut0KD6YTPbs2eN669atrpkUyRRHtd+Bs2HDBtcTJkzI8t85N5UtWzY5J5aGJLLm8t87duzoWv31wKCVtF+/fq45NpiZ3X333a45vuYXWNyWxcJpEe/cubPrdLsv0oqTEEIIIYQQQsSgBychhBBCCCGEiCGjrHpc2h86dKhrLq/++c9/Dj7DVL1MgsXvnnvuueA1pu0wSZDpb+m2dJosogmFLHr73XffuT755JNdJyO5Jr9A69nYsWNd16pVy3WzZs2Sek6pBm26tEmsXr3adadOnVyXL1/eta77ECZncX6pV6+eaya+qf0OnDlz5rimjZyphK1bt07qOaUrLAo+cuRI1yxszetdxMOkzNmzZ7tmOnH0PpJW6PwIC+gOGzbMNRNvTzvttKSeU06iFSchhBBCCCGEiEEPTkIIIYQQQggRgx6chBBCCCGEECKGtN/jxJhNenq5r+faa6913a5du+Dzhx9+eC6eXd7xxRdfuKYX18ysdOnSruk51V6ceLZv3x78zP1i9JEzMleRr9mD+/U+/vhj1yw5cNttt7kuXrx4ck4sRWGENiPyuUfnqKOOcs047fwO94eZhW02depU1y1btnTN617sH4wrXrt2rWvuYeRcXaZMmeScWJozbdo014zI5jwfLcki9g33yI8aNco199VeffXVwWfy+5gwaNAg14sXL3b96KOPuubeMe5nTAe04iSEEEIIIYQQMejBSQghhBBCCCFiSK/1sSygXYdWPVqlzj33XNdFixZNzonlAcuXL3d95513uv7d78LnY1YOZzvRpqJY3f/BdmE1drPQZsJ4Tdkhco5du3a5Zvv//PPPrmmdyu988803rhmnzWta/XP/oKWR1zpJN5tJKsBofFogOT/37NnTtfprYlgGY8CAAa5Llizpum3btq5p5ZWNPGtYdoRx2p999plr9s8GDRoEn8+P7cr5eObMmVn++5FHHumakeXphlachBBCCCGEECIGPTgJIYQQQgghRAxp6THgUvPTTz/tesqUKa5fffVV13Xr1nWdaRY0Juc8+OCDrpctW+b6gQceCD7TtWtX11w6FVlDC+SIESOC12rUqOGa6Y3FihXL/RPLYGiPZLIZE3poPznuuOOSc2IpCK0QZmEK1IQJE1w3adLENa2NUSuv+B9PPPGE68KFC7umLZfJrvnRorO/sJ+yj37wwQeuy5cv7/qEE05IzomlOUzSY4LuJZdc4pq2Ms358WzatMn1pEmTXPP+8ZRTTnFdpEiR5JxYCrNu3TrXtIu3adPGdb169Vyn81ipGVMIIYQQQgghYtCDkxBCCCGEEELEkJZWPS4DMmWLy6Wnn36660yzotCqOGvWLNcfffSRa9rGOnfuHHxeS/UHxjvvvON61apVwWu33HKL6zPOOMM107jEgUP70/vvv+/6hx9+cN27d2/XmVrIen9ggWCzMPlpx44drlmkMZPty9nh66+/Dn7+8ssvXZ933nmuL774YtfpnA6VTH788UfX27Ztc71x40bX9evXd01rpEjMSy+95Jpt1qNHD9fHHHNMEs8oPeHc8tBDD7mePn26a9qdK1Wq5Do/jqHRYuG03y5atMg1EweZ9JjOZNYThRBCCCGEEELkAnpwEkIIIYQQQogY0saqt3v3btc33nijayZuXXfdda4z2boze/Zs1yx4V6JECde0khx99NHJObEMgv1tzJgxrmlxMjNr166dayXr5By09TzzzDOu2cbVqlVL6jmlKlHLxJAhQ1w3bdrUdatWrVync6JRbjJ58uTgZ9p3Wrdu7bpUqVJJO6dMgfad1157zTXn6qpVq7pmEVIVwA3Zvn276zlz5ri+4IILXDOVMNO2K+QG48aNcz1w4EDXZ599tmta8ytWrOg6P1r19u7dG/y8c+dO1yxwzQTSQoUK5f6JJQFdTUIIIYQQQggRgx6chBBCCCGEECKGlLXqRe0ngwYNcs2CeX379nXNIm+ZvDTN4my0MLDgXa1atZJ6TpkGiynTYnLrrbcG7ytbtqzr/Lhcn1ts2bLFNYs5N27c2LWskf+F/dMsLNh8zz33uFbSY9YwpXTXrl3Ba0wjO+mkk1zLOhYPk/PMwnQy2suuuOIK13fccYdrtXFi3nrrLde8H2BBdrVfPLQ83nvvva6LFy/u+m9/+5vrE0880XUm32PuD7TTm5nNmzfPNbc60J7HsTadt9Pk729eCCGEEEIIIfYDPTgJIYQQQgghRAwpZdWjPY/LfmZhetz555/vuk+fPq6ZKpfJtqn27dtnqbl0qmX67PHhhx+6psWpXr16wfvSebk51eD1/+9//zvL99CCqtSt/zJ8+PDg5ypVqmSpo/Zn8V9o16FF1yxM1StTpkzSzikTWLNmTfDzqFGjXJcuXdo1i7XLfrt/fPPNN65PPvlk13Xq1HFdsGDBpJ5TOhAtFj5+/HjXtDg/8MADrmvWrOk6v9vzSNTWzDRXFr295pprXGfK/ZJ6gRBCCCGEEELEoAcnIYQQQgghhIhBD05CCCGEEEIIEUNK7XFav3696xdffDF4jd7n3r17uy5ZsmTun1iakJ/3eeQ0jBnv0qWLa0YSm5kdcsghSTunTIf7GrkfolOnTq4ZDav+/l/q1q0b/Mx9TZUrV3atvpo1xYoVc809TWZh1fsKFSok7ZzSlb1797reuHFj8NpXX33l+qqrrnLNCO1M3puck3Bvc7NmzVwrLnvfsLSFmdnTTz/tmuMmy14cemhK3SanDB9//HHwM/c8de/e3XWlSpWSdUpJQ1eWEEIIIYQQQsSgBychhBBCCCGEiCGl1iAXLVrkeu7cucFrXJrm0r4QucGNN96Y16eQ72Cs9s8//+z66quvdl2uXLmknlM60LVr17w+hbSGlqaHH344eI2WXRHP5s2bXY8bNy54bffu3a5btmzpWlaoA4eWPLH/sISFWWgjo+W5atWqyTqltILz8tixY4PX2CfZlpkYi68VJyGEEEIIIYSIQQ9OQgghhBBCCBFDSq2R16pVy3WfPn2C16pXr+66aNGiSTsnIUTuEK3iPmHCBNetWrVyfcopp7g+7LDDcv28RP5FVtDswWt68eLFwWtMJI2mkwqRDDiXmJk9+eSTrmlDUwJp1mzatMn11q1bg9eY4Mq060xMytSKkxBCCCGEEELEoAcnIYQQQgghhIihwK+//prX5yCEEEIIIYQQKY1WnIQQQgghhBAiBj04CSGEEEIIIUQMenASQgghhBBCiBj04CSEEEIIIYQQMejBSQghhBBCCCFi0IOTEEIIIYQQQsTwfyCNhYErFqkiAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 1080x324 with 30 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "X_train_random = X_train[25,].reshape((1,28,28,1))\n",
    "Y_train_random = Y_train[25,].reshape((1,10))\n",
    "\n",
    "plt.figure(figsize=(15,4.5))\n",
    "\n",
    "for i in range(30):  \n",
    "    plt.subplot(3, 10, i+1)\n",
    "    X_train_augmented, Y_train_augmented = datagen.flow(X_train_random,Y_train_random).next()\n",
    "    plt.imshow(X_train_augmented[0].reshape((28,28)),cmap=plt.cm.binary)\n",
    "    plt.axis('off')\n",
    "    # Now for the second and third row take 2 different digit\n",
    "    # 9 and 7 respectively\n",
    "    # And because I am printing 9 columns for the same image, I have to \n",
    "    # advance the 'i' value by 10\n",
    "    if i==9:\n",
    "        X_train_random = X_train[11,].reshape((1,28,28,1))\n",
    "    if i==19:\n",
    "        X_train_random = X_train[18,].reshape((1,28,28,1))\n",
    "plt.subplots_adjust(wspace=-0.1, hspace=-0.1)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff0bd3f3",
   "metadata": {
    "_uuid": "9ea116cd3688cb26ac79b9fecc7309a1aebf3b63",
    "papermill": {
     "duration": 0.033565,
     "end_time": "2022-04-19T13:17:10.899503",
     "exception": false,
     "start_time": "2022-04-19T13:17:10.865938",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "---\n",
    "\n",
    "### Build 15 Convolutional Neural Networks with bagging\n",
    "\n",
    "### LeNet Architectural\n",
    "![Imgur](https://imgur.com/by2Dc72.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "56cbe0e2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:10.978293Z",
     "iopub.status.busy": "2022-04-19T13:17:10.974049Z",
     "iopub.status.idle": "2022-04-19T13:17:10.980329Z",
     "shell.execute_reply": "2022-04-19T13:17:10.980705Z",
     "shell.execute_reply.started": "2022-04-19T13:08:55.805015Z"
    },
    "papermill": {
     "duration": 0.048153,
     "end_time": "2022-04-19T13:17:10.980859",
     "exception": false,
     "start_time": "2022-04-19T13:17:10.932706",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def build_model(num_cnn):\n",
    "    model = [0] *num_cnn\n",
    "    for j in range(num_cnn):\n",
    "        model[j] = Sequential()\n",
    "\n",
    "        model[j].add(Conv2D(32, kernel_size = 3, activation='relu', input_shape = (28, 28, 1))) # Two stacked 3x3 filters instead of single 5x5 filters. \n",
    "        model[j].add(BatchNormalization()) # Batch normalization is added\n",
    "        model[j].add(Conv2D(32, kernel_size = 3, activation='relu')) # and ReLU activation replaces sigmoid.\n",
    "        model[j].add(BatchNormalization())\n",
    "        model[j].add(Conv2D(32, kernel_size = 5, strides=2, padding='same', activation='relu')) # Stride 2 replaces pooling layers. These become learnable pooling layers.\n",
    "        model[j].add(BatchNormalization())\n",
    "        model[j].add(Dropout(0.4)) #  Dropout is added\n",
    "\n",
    "        model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n",
    "        model[j].add(BatchNormalization())\n",
    "        model[j].add(Conv2D(64, kernel_size = 3, activation='relu'))\n",
    "        model[j].add(BatchNormalization())\n",
    "        model[j].add(Conv2D(64, kernel_size = 5, strides=2, padding='same', activation='relu'))\n",
    "        model[j].add(BatchNormalization())\n",
    "        model[j].add(Dropout(0.4))\n",
    "\n",
    "        model[j].add(Conv2D(128, kernel_size = 4, activation='relu'))\n",
    "        model[j].add(BatchNormalization())\n",
    "        model[j].add(Flatten())\n",
    "        model[j].add(Dropout(0.4))\n",
    "        model[j].add(Dense(10, activation='softmax'))\n",
    "\n",
    "        model[j].compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])\n",
    "    return model\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "48a5f8a9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:11.050217Z",
     "iopub.status.busy": "2022-04-19T13:17:11.049666Z",
     "iopub.status.idle": "2022-04-19T13:17:13.431510Z",
     "shell.execute_reply": "2022-04-19T13:17:13.432094Z",
     "shell.execute_reply.started": "2022-04-19T13:08:55.820993Z"
    },
    "papermill": {
     "duration": 2.418587,
     "end_time": "2022-04-19T13:17:13.432303",
     "exception": false,
     "start_time": "2022-04-19T13:17:11.013716",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-19 13:17:11.137473: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-04-19 13:17:11.226537: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-04-19 13:17:11.227317: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-04-19 13:17:11.228457: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-04-19 13:17:11.229443: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-04-19 13:17:11.230200: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-04-19 13:17:11.230873: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-04-19 13:17:12.956703: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-04-19 13:17:12.957665: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-04-19 13:17:12.958348: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-04-19 13:17:12.958950: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 15403 MB memory:  -> device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\n"
     ]
    }
   ],
   "source": [
    "# num_cnn=1 # First running for 1 num_cnn only to check all is working fine\n",
    "num_cnn=10\n",
    "model = build_model(num_cnn)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e75b8d76",
   "metadata": {
    "_uuid": "e433661c7762b947c0fbfc4ad3f5e1d2e056312c",
    "papermill": {
     "duration": 0.033901,
     "end_time": "2022-04-19T13:17:13.501482",
     "exception": false,
     "start_time": "2022-04-19T13:17:13.467581",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Train 10 CNNs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e2656098",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:13.577057Z",
     "iopub.status.busy": "2022-04-19T13:17:13.574553Z",
     "iopub.status.idle": "2022-04-19T13:17:13.579022Z",
     "shell.execute_reply": "2022-04-19T13:17:13.579434Z",
     "shell.execute_reply.started": "2022-04-19T13:08:58.117756Z"
    },
    "papermill": {
     "duration": 0.043829,
     "end_time": "2022-04-19T13:17:13.579554",
     "exception": false,
     "start_time": "2022-04-19T13:17:13.535725",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train(num_epochs):\n",
    "    # Keep reducing the Learning Rate\n",
    "    lr_scheduler = LearningRateScheduler(lambda x: 1e-3 * 0.95 ** x)\n",
    "    # Training starts\n",
    "    history = [0] * num_cnn\n",
    "    for cnn in range(num_cnn):\n",
    "        X_train_split, X_val_split, Y_train_split, Y_val_split = train_test_split(\n",
    "            X_train, Y_train, test_size=0.1\n",
    "        )\n",
    "        history[cnn] = model[cnn].fit(\n",
    "            datagen.flow(X_train_split, Y_train_split, batch_size=64),\n",
    "            epochs=num_epochs,\n",
    "            steps_per_epoch=X_train_split.shape[0] // 64,\n",
    "            validation_data=(X_val_split, Y_val_split),\n",
    "            callbacks=[lr_scheduler],\n",
    "            verbose=0,\n",
    "        )\n",
    "        print(\n",
    "            \"CNN {0:d}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}\".format(\n",
    "                cnn + 1,\n",
    "                num_epochs,\n",
    "                max(history[cnn].history[\"accuracy\"]),\n",
    "                max(history[cnn].history[\"val_accuracy\"]),\n",
    "            )\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "4087c72b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:13.652714Z",
     "iopub.status.busy": "2022-04-19T13:17:13.651013Z",
     "iopub.status.idle": "2022-04-19T13:17:35.505449Z",
     "shell.execute_reply": "2022-04-19T13:17:35.504736Z",
     "shell.execute_reply.started": "2022-04-19T13:08:58.128390Z"
    },
    "papermill": {
     "duration": 21.892206,
     "end_time": "2022-04-19T13:17:35.505633",
     "exception": false,
     "start_time": "2022-04-19T13:17:13.613427",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-04-19 13:17:14.167958: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n",
      "2022-04-19 13:17:15.731207: I tensorflow/stream_executor/cuda/cuda_dnn.cc:369] Loaded cuDNN version 8005\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN 1: Epochs=1, Train accuracy=0.85552, Validation accuracy=0.96452\n"
     ]
    }
   ],
   "source": [
    "# num_epochs = 1  # First running for 1 epoch only to check all is working fine\n",
    "num_epochs = 45\n",
    "train(num_epochs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08880636",
   "metadata": {
    "papermill": {
     "duration": 0.036261,
     "end_time": "2022-04-19T13:17:35.579910",
     "exception": false,
     "start_time": "2022-04-19T13:17:35.543649",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Total image count\n",
    "\n",
    "42,000 number of samples images * 90% Train Split Dataset * 45 epochs * 10 CNN => 17 Million\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "076ebed2",
   "metadata": {
    "papermill": {
     "duration": 0.040946,
     "end_time": "2022-04-19T13:17:35.656807",
     "exception": false,
     "start_time": "2022-04-19T13:17:35.615861",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## [LearningRateScheduler](https://keras.io/api/callbacks/learning_rate_scheduler/)\n",
    "\n",
    "\n",
    "`tf.keras.callbacks.LearningRateScheduler(schedule, verbose=0)`\n",
    "\n",
    "At the beginning of every epoch, this callback gets the updated learning rate value from schedule function provided at `__init__`, with the current epoch and current learning rate, and applies the updated learning rate on the optimizer.\n",
    "\n",
    "Arguments\n",
    "\n",
    "schedule: a function that takes an epoch index (integer, indexed from 0) and current learning rate (float) as inputs and returns a new learning rate as output (float).\n",
    "verbose: int. 0: quiet, 1: update messages.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ebb97b4",
   "metadata": {
    "_uuid": "28b78b6502d2d1c993555725383f3e30728fa5be",
    "papermill": {
     "duration": 0.044467,
     "end_time": "2022-04-19T13:17:35.740582",
     "exception": false,
     "start_time": "2022-04-19T13:17:35.696115",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Ensemble 10 CNN predictions and submit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "120c7c84",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:35.827043Z",
     "iopub.status.busy": "2022-04-19T13:17:35.825742Z",
     "iopub.status.idle": "2022-04-19T13:17:35.829463Z",
     "shell.execute_reply": "2022-04-19T13:17:35.829000Z",
     "shell.execute_reply.started": "2022-04-19T13:09:20.457789Z"
    },
    "papermill": {
     "duration": 0.045971,
     "end_time": "2022-04-19T13:17:35.829571",
     "exception": false,
     "start_time": "2022-04-19T13:17:35.783600",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28000, 28, 28, 1)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "40a20ed8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:35.907735Z",
     "iopub.status.busy": "2022-04-19T13:17:35.906852Z",
     "iopub.status.idle": "2022-04-19T13:17:35.910803Z",
     "shell.execute_reply": "2022-04-19T13:17:35.910358Z",
     "shell.execute_reply.started": "2022-04-19T13:10:01.295560Z"
    },
    "papermill": {
     "duration": 0.045138,
     "end_time": "2022-04-19T13:17:35.910920",
     "exception": false,
     "start_time": "2022-04-19T13:17:35.865782",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28000, 10)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_ensemble_prediction = np.zeros( (X_test.shape[0],10) )\n",
    "\n",
    "final_ensemble_prediction.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e0560c0",
   "metadata": {
    "papermill": {
     "duration": 0.035791,
     "end_time": "2022-04-19T13:17:35.984293",
     "exception": false,
     "start_time": "2022-04-19T13:17:35.948502",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Final ensemble"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fa78b511",
   "metadata": {
    "_uuid": "6e4e01ffe692c34c555bdbf5d606611f9a128b9c",
    "execution": {
     "iopub.execute_input": "2022-04-19T13:17:36.064501Z",
     "iopub.status.busy": "2022-04-19T13:17:36.063559Z",
     "iopub.status.idle": "2022-04-19T13:17:37.980719Z",
     "shell.execute_reply": "2022-04-19T13:17:37.981506Z",
     "shell.execute_reply.started": "2022-04-19T13:10:03.916792Z"
    },
    "papermill": {
     "duration": 1.961103,
     "end_time": "2022-04-19T13:17:37.981714",
     "exception": false,
     "start_time": "2022-04-19T13:17:36.020611",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "success\n"
     ]
    }
   ],
   "source": [
    "for j in range(num_cnn):\n",
    "    final_ensemble_prediction = np.zeros( (X_test.shape[0],10) )\n",
    "    final_ensemble_prediction = final_ensemble_prediction + model[j].predict(X_test)\n",
    "\n",
    "    final_ensemble_prediction = np.argmax(final_ensemble_prediction,axis = 1)\n",
    "    \n",
    "#     print(\"final_ensemble_prediction.shape \", final_ensemble_prediction.shape)\n",
    "\n",
    "    final_ensemble_prediction = pd.Series(final_ensemble_prediction,name=\"Label\")\n",
    "\n",
    "    submission = pd.concat([pd.Series(range(1,28001),name = \"ImageId\"),final_ensemble_prediction],axis = 1)\n",
    "\n",
    "    submission.to_csv(\"MNIST_ensemble_cnn.csv\",index=False)\n",
    "    \n",
    "    print('success')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "058df107",
   "metadata": {
    "papermill": {
     "duration": 0.036264,
     "end_time": "2022-04-19T13:17:38.057036",
     "exception": false,
     "start_time": "2022-04-19T13:17:38.020772",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### Some References\n",
    "\n",
    "1. https://www.kaggle.com/code/friskycodeur/image-recognition-using-cnn-explained/notebook\n",
    "2. https://www.kaggle.com/code/mohammedmurtuzalabib/mnist-ensemble-of-5-cnns-0-99742/notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4dee98ca",
   "metadata": {
    "papermill": {
     "duration": 0.035873,
     "end_time": "2022-04-19T13:17:38.129027",
     "exception": false,
     "start_time": "2022-04-19T13:17:38.093154",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Bonus Point - A Note on `model[cnn].fit_` method used inside my train() method above\n",
    "\n",
    "As mentioned in the [documentation][1] : \n",
    "\n",
    ">  x: Input data. It could be\n",
    ">  - A Numpy array (or array-like), or a list of arrays (in case the model has multiple inputs). \n",
    ">  - A TensorFlow tensor, or a list of tensors (in case the model has multiple inputs).\n",
    ">  - A dict mapping input names to the corresponding array/tensors, if the\n",
    ">      model has named inputs. \n",
    ">  - A tf.data dataset. Should return a tuple of either (inputs, targets) or (inputs, targets, sample_weights)\n",
    ">  - *A generator or keras.utils.Sequence returning (inputs, targets) or (inputs, targets, sample weights).\n",
    "\n",
    "\n",
    "you can simply pass the generator to `Model.fit` as below\n",
    "\n",
    "```py\n",
    "\n",
    "data_gen_train = ImageDataGenerator(rescale=1/255.)\n",
    "\n",
    "data_gen_valid = ImageDataGenerator(rescale=1/255.)\n",
    "\n",
    "train_generator = data_gen_train.flow_from_directory(train_dir, target_size=(128,128), batch_size=128, class_mode=\"binary\")\n",
    "\n",
    "valid_generator = data_gen_valid.flow_from_directory(validation_dir, target_size=(128,128), batch_size=128, class_mode=\"binary\")\n",
    "\n",
    "model.fit(train_generator, epochs=2, validation_data=valid_generator) \n",
    "\n",
    "```\n",
    "\n",
    "  [1]: https://www.tensorflow.org/versions/r2.1/api_docs/python/tf/keras/Model#fit\n",
    "\n",
    "\n",
    "---"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 53.653053,
   "end_time": "2022-04-19T13:17:41.201685",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-04-19T13:16:47.548632",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}