deep_03.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 행렬의 곱"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "a = np.array([1,2,3])\n",
    "b = np.array([4,5,6])  # [4, 10, 18]\n",
    "# c = np.sum(a*b)\n",
    "c = np.dot(a,b) # (3,)(3,)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 3)\n",
      "(3, 1)\n",
      "(1, 1)\n",
      "2\n",
      "[[12]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "a = np.array([[1,2,3]]) \n",
    "print(a.shape)  # (1, 3)\n",
    "b = np.array([[2],[2],[2]]) \n",
    "print(b.shape)  # (3, 1)\n",
    "c = np.dot(a,b) # (1,3)(3,1)\n",
    "print(c.shape)   \n",
    "print(c.ndim)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 3)\n",
      "(3, 1)\n",
      "(2, 1)\n",
      "2\n",
      "[[12]\n",
      " [30]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "a = np.array([[1,2,3],\n",
    "              [4,5,6]]) \n",
    "print(a.shape)  # (2, 3)\n",
    "b = np.array([[2],\n",
    "              [2],\n",
    "              [2]]) \n",
    "print(b.shape)  # (3, 1)\n",
    "c = np.dot(a,b)  # (2,3)(3,1)\n",
    "print(c.shape)   \n",
    "print(c.ndim)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3,)\n",
      "(3, 2)\n",
      "(2,)\n",
      "[14 14]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "a = np.array([1,2,3]) # 1차원 배열\n",
    "print(a.shape)   # (3,)\n",
    "b = np.array([[1,1],\n",
    "              [2,2],\n",
    "              [3,3]]) # 2차원 배열\n",
    "print(b.shape)  # (3,2)\n",
    "c = np.dot(a,b)\n",
    "print(c.shape)   # (3,)(3,2) = > (2,)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 2)\n",
      "(2,)\n",
      "(3,)\n",
      "[2 4 6]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "a = np.array([[1,1],\n",
    "              [2,2],\n",
    "              [3,3]]) # 2차원 배열\n",
    "print(a.shape)        # (3, 2)\n",
    "b = np.array([1,1])   # 1차원 배열\n",
    "print(b.shape)  # (2,)\n",
    "c = np.dot(a,b)\n",
    "print(c.shape)   # (3,2)(2) = > (3,)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2, 2)\n",
      "(2, 2)\n",
      "(2, 2)\n",
      "[[4 4]\n",
      " [8 8]]\n",
      "(2, 2)\n",
      "[[6 6]\n",
      " [6 6]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "a = np.array([[1,1],\n",
    "              [2,2]]) # 2차원 배열\n",
    "print(a.shape)        # (2, 2)\n",
    "b = np.array([[2,2],\n",
    "              [2,2]]) # 2차원 배열\n",
    "print(b.shape)  # (2, 2)\n",
    "c = np.dot(a,b)\n",
    "print(c.shape)   # (2,2)(2,2) = > (2,2)\n",
    "print(c)\n",
    "\n",
    "c = np.dot(b,a)\n",
    "print(c.shape)   # (2,2)(2,2) = > (2,2)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 2)\n",
      "(2, 1)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "shapes (2,1) and (3,2) not aligned: 1 (dim 1) != 3 (dim 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-ee13377672a3>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     12\u001b[0m \u001b[1;31m# c = np.dot(a,b)  # (3,2)(2,1)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 13\u001b[1;33m \u001b[0mc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mb\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0ma\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# (2,1)(3,2)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     14\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mc\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mdot\u001b[1;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: shapes (2,1) and (3,2) not aligned: 1 (dim 1) != 3 (dim 0)"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "a = np.array([[1,1],\n",
    "              [2,2],\n",
    "              [3,3]]) # 2차원 배열\n",
    "print(a.shape)        # (3, 2)\n",
    "\n",
    "b = np.array([[2],\n",
    "              [2]]) # 2차원 배열\n",
    "print(b.shape)        # (2, 1)\n",
    "\n",
    "# c = np.dot(a,b)  # (3,2)(2,1)\n",
    "c = np.dot(b,a)  # (2,1)(3,2)\n",
    "print(c.shape)\n",
    "print(c)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 행렬의 전치"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3,)\n",
      "(3,)\n",
      "[1 2 3]\n"
     ]
    }
   ],
   "source": [
    "a = np.array([1,2,3])\n",
    "print(a.shape)  # (3,)\n",
    "\n",
    "print(a.T.shape) # (3,)\n",
    "print(a.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 3)\n",
      "(3, 1)\n",
      "[[1]\n",
      " [2]\n",
      " [3]]\n"
     ]
    }
   ],
   "source": [
    "a = np.array([[1,2,3]])\n",
    "print(a.shape)  # (1,3)\n",
    "\n",
    "print(a.T.shape) # (3,1)\n",
    "print(a.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3,)\n",
      "[[1]\n",
      " [2]\n",
      " [3]]\n",
      "(3, 1)\n"
     ]
    }
   ],
   "source": [
    "a = np.array([1,2,3])      # (3,)\n",
    "print(a.shape)\n",
    "# b = np.reshape(a, (3,1))   # (3,1)\n",
    "b = np.reshape(a, (-1,1))    # (3,1)\n",
    "print(b)\n",
    "print(b.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3,)\n",
      "[[1]\n",
      " [2]\n",
      " [3]]\n",
      "(3, 1)\n"
     ]
    }
   ],
   "source": [
    "a = np.array([1,2,3])        \n",
    "print(a.shape)\n",
    "b = np.reshape(a, (-1,1))    \n",
    "print(b)\n",
    "print(b.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 2 3]\n",
      " [1 2 3]]\n",
      "(2, 3)\n",
      "(3, 2)\n",
      "[[1 1]\n",
      " [2 2]\n",
      " [3 3]]\n"
     ]
    }
   ],
   "source": [
    "a = np.array([[1,2,3],\n",
    "              [1,2,3]])\n",
    "print(a)\n",
    "print(a.shape)        # (2, 3)\n",
    "# print(a.T.shape)      # (3, 2)\n",
    "# print(a.T)\n",
    "# b = np.transpose( a, (1,0))\n",
    "# print(b.shape)\n",
    "# print(b)\n",
    "b = np.swapaxes(a, 0,1)\n",
    "print(b.shape)\n",
    "print(b)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  배치 경사 하강 구현"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(364, 30)\n",
      "(364,)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "from sklearn.model_selection import train_test_split\n",
    "cancer = load_breast_cancer()\n",
    "x = cancer.data\n",
    "y = cancer.target\n",
    "x_train_all, x_test, y_train_all, y_test = train_test_split(x, y, stratify=y, \n",
    "                                                            test_size=0.2, random_state=42)\n",
    "x_train, x_val, y_train, y_val = train_test_split(x_train_all, y_train_all, stratify=y_train_all, \n",
    "                                                  test_size=0.2, random_state=42)\n",
    "print(x_train.shape)\n",
    "print(y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SingleLayer:    \n",
    "    def __init__(self, learning_rate=0.1, l1=0, l2=0):\n",
    "        self.w = None              # 가중치\n",
    "        self.b = None              # 절편\n",
    "        self.losses = []           # 훈련 손실\n",
    "        self.val_losses = []       # 검증 손실\n",
    "        self.w_history = []        # 가중치 기록\n",
    "        self.lr = learning_rate    # 학습률\n",
    "        self.l1 = l1               # L1 손실 하이퍼파라미터\n",
    "        self.l2 = l2               # L2 손실 하이퍼파라미터\n",
    "\n",
    "    def forpass(self, x):\n",
    "#         print(\"forpass()\")\n",
    "#         print(x.shape)       # (364,30)\n",
    "#         print(self.w.shape)  # (30, 1)\n",
    "#         print(self.b)        # 0\n",
    "        z = np.dot(x, self.w) + self.b        # 선형 출력을 계산합니다.\n",
    "            # (364,30)(30,1)\n",
    "#         print(z.shape)       # (364,1)\n",
    "        return z\n",
    "\n",
    "    def backprop(self, x, err):\n",
    "#         print(\"backprop()\")\n",
    "#         print(x.shape)      # (364,30)   \n",
    "#         print(err.shape)    # (364,1)\n",
    "        m = len(x)          # 364\n",
    "        temp = np.dot(x.T, err)\n",
    "        \n",
    "#         print(temp.shape)   # (30,1)  \n",
    "#         print(temp)\n",
    "#         print(temp/m)\n",
    "        w_grad = np.dot(x.T, err) / m         # 가중치에 대한 그래디언트를 계산합니다.\n",
    "        b_grad = np.sum(err) / m              # 절편에 대한 그래디언트를 계산합니다.\n",
    "        return w_grad, b_grad\n",
    "\n",
    "    def activation(self, z):\n",
    "        z = np.clip(z, -100, None)            # 안전한 np.exp() 계산을 위해\n",
    "        a = 1 / (1 + np.exp(-z))              # 시그모이드 계산\n",
    "        return a\n",
    "        \n",
    "    def fit(self, x, y, epochs=100, x_val=None, y_val=None):\n",
    "        print(x.shape)   # (364,30)\n",
    "        print(y.shape)   # (364,)\n",
    "        y = y.reshape(-1, 1)                  # 타깃을 열 벡터로 바꿉니다.\n",
    "        print(y.shape)   # (364,1)\n",
    "        y_val = y_val.reshape(-1, 1)\n",
    "        m = len(x)                            # 샘플 개수를 저장합니다.\n",
    "        print(m)         # 364\n",
    "        self.w = np.ones((x.shape[1], 1))     # 가중치를 초기화합니다.\n",
    "        print(self.w.shape)  # (30, 1)\n",
    "\n",
    "        self.b = 0                            # 절편을 초기화합니다.\n",
    "        self.w_history.append(self.w.copy())  # 가중치를 기록합니다.\n",
    "        # epochs만큼 반복합니다.\n",
    "        for i in range(epochs):\n",
    "            z = self.forpass(x)               # 정방향 계산을 수행합니다.\n",
    "#             print(z[:10,:])\n",
    "            a = self.activation(z)            # 활성화 함수를 적용합니다.\n",
    "#             print(a.shape)\n",
    "#             print(a[:10,:])\n",
    "           \n",
    "            err = a - y                   # 오차를 계산합니다.\n",
    "            #    (364,1)-(364,1)\n",
    "#             print(err.shape)  # (364,1)\n",
    "           \n",
    "            # 오차를 역전파하여 그래디언트를 계산합니다.\n",
    "            w_grad, b_grad = self.backprop(x, err)\n",
    "#             break\n",
    "            # 그래디언트에서 페널티 항의 미분 값을 더합니다.\n",
    "            w_grad += (self.l1 * np.sign(self.w) + self.l2 * self.w) / m\n",
    "            # 가중치와 절편을 업데이트합니다.\n",
    "            self.w -= self.lr * w_grad\n",
    "            self.b -= self.lr * b_grad\n",
    "            # 가중치를 기록합니다.\n",
    "            self.w_history.append(self.w.copy())\n",
    "            # 안전한 로그 계산을 위해 클리핑합니다.\n",
    "            a = np.clip(a, 1e-10, 1-1e-10)\n",
    "            # 로그 손실과 규제 손실을 더하여 리스트에 추가합니다.\n",
    "            loss = np.sum(-(y*np.log(a) + (1-y)*np.log(1-a)))\n",
    "            self.losses.append((loss + self.reg_loss()) / m)\n",
    "            # 검증 세트에 대한 손실을 계산합니다.\n",
    "            self.update_val_loss(x_val, y_val)\n",
    "    \n",
    "    def predict(self, x):\n",
    "        z = self.forpass(x)      # 정방향 계산을 수행합니다.\n",
    "        return z > 0             # 스텝 함수를 적용합니다.\n",
    "    \n",
    "    def score(self, x, y):\n",
    "        # 예측과 타깃 열 벡터를 비교하여 True의 비율을 반환합니다.\n",
    "        return np.mean(self.predict(x) == y.reshape(-1, 1))\n",
    "    \n",
    "    def reg_loss(self):\n",
    "        # 가중치에 규제를 적용합니다.\n",
    "        return self.l1 * np.sum(np.abs(self.w)) + self.l2 / 2 * np.sum(self.w**2)\n",
    "    \n",
    "    def update_val_loss(self, x_val, y_val):\n",
    "        z = self.forpass(x_val)            # 정방향 계산을 수행합니다.\n",
    "        a = self.activation(z)             # 활성화 함수를 적용합니다.\n",
    "        a = np.clip(a, 1e-10, 1-1e-10)     # 출력 값을 클리핑합니다.\n",
    "        # 로그 손실과 규제 손실을 더하여 리스트에 추가합니다.\n",
    "        val_loss = np.sum(-(y_val*np.log(a) + (1-y_val)*np.log(1-a)))\n",
    "        self.val_losses.append((val_loss + self.reg_loss()) / len(y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(x_train)\n",
    "x_train_scaled = scaler.transform(x_train)\n",
    "x_val_scaled = scaler.transform(x_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(364, 30)\n",
      "(364,)\n",
      "(364, 1)\n",
      "364\n",
      "(30, 1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.978021978021978"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "single_layer = SingleLayer(l2=0.01)\n",
    "single_layer.fit(x_train_scaled, y_train, \n",
    "                 x_val=x_val_scaled, y_val=y_val, epochs=1000)\n",
    "single_layer.score(x_val_scaled, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAt5ElEQVR4nO3deZiU5Z3v//e39l6haaCBZhcUURQRBUOiMZ4oLhEnJkriPvHyGHVinJ9GPBNzJpnknMw52SbzU4lJTMZo4o4hCQkmJsa4swTZZJelQaFpaOi1uqr6Pn88D1A01VC9VBd0f17XVdez3tXfu0U+3M9qzjlERETaCuS7ABEROT4pIEREJCMFhIiIZKSAEBGRjBQQIiKSkQJCREQyymlAmNlMM1trZhvMbE6G7bPMbLmZLTOzxWb20WzbiohIblmu7oMwsyCwDvgkUAUsAj7nnFudtk8x0OCcc2Z2BvCMc25CNm1FRCS3cjmCOBfY4Jzb5JxrAZ4CZqXv4Jyrd4cSqghw2bYVEZHcCuXwuyuBbWnLVcC0tjuZ2T8A/xsYDFzekbZ++9uA2wCKiorOnjBhQpcLP8KeTTQ1N1NXchKDS6Ld//0iInmyZMmS3c65QZm25TIgLMO6I45nOefmAfPM7Hzg34D/lm1bv/2jwKMAU6dOdYsXL+50we1xT32eNauX8/uPPc8/f/Lkbv9+EZF8MbMt7W3L5SGmKmBE2vJwYEd7OzvnXgVOMrOBHW2baxaMELYUyVRrvkoQEelxuQyIRcB4MxtjZhFgNjA/fQczG2dm5s9PASJATTZte1QgTIQUyVY92FBE+o6cHWJyziXN7C5gIRAEHnPOrTKz2/3tc4GrgRvNLAE0Adf6J60zts1VrccUDBO2JC1JjSBEpO/I5TkInHMLgAVt1s1Nm/934N+zbZs3wTAhUiRbFRAiPS2RSFBVVUVzc3O+SzmhxWIxhg8fTjgczrpNTgOi1wiEiZAkmdIhJpGeVlVVRUlJCaNHj8Y/Ii0d5JyjpqaGqqoqxowZk3U7PWojG/4IokUnqUV6XHNzM+Xl5QqHLjAzysvLOzwKU0BkIxgmrBGESN4oHLquM79DBUQ2AmGCOgchIn2MAiIbwQhBWkkkEvmuRESkxyggshH0zuWnkgoIkb6mtraWhx9+uMPtLrvsMmprazvc7uabb+a5557rcLtcUEBkI+BdFpZKtOS5EBHpae0FRCqVOmq7BQsW0L9//xxV1TN0mWs2ghEAXEoBIZJPX//NKlbv2N+t3zlxWCn/81Ontbt9zpw5bNy4kcmTJxMOhykuLmbo0KEsW7aM1atXc9VVV7Ft2zaam5u5++67ue222wAYPXo0ixcvpr6+nksvvZSPfvSjvPHGG1RWVvLrX/+agoKCY9b28ssvc++995JMJjnnnHN45JFHiEajzJkzh/nz5xMKhbj44ov5zne+w7PPPsvXv/51gsEg/fr149VXX+3y70YBkY2Q9wRXl4znuRAR6Wnf/va3WblyJcuWLeOVV17h8ssvZ+XKlQfvJ3jssccYMGAATU1NnHPOOVx99dWUl5cf9h3r16/nV7/6FT/+8Y+55ppreP7557n++uuP+nObm5u5+eabefnllzn55JO58cYbeeSRR7jxxhuZN28ea9aswcwOHsb6xje+wcKFC6msrOzUoa1MFBDZCMW8qQJCJK+O9i/9nnLuuecedrPZD3/4Q+bNmwfAtm3bWL9+/REBMWbMGCZPngzA2WefzebNm4/5c9auXcuYMWM4+WTvCdI33XQTDz30EHfddRexWIxbb72Vyy+/nCuuuAKAGTNmcPPNN3PNNdfw6U9/uht6qnMQ2fFHEAoIESkqKjo4/8orr/CnP/2JN998k3fffZezzjor481o0eih98gEg0GSyeQxf057b/sMhUK88847XH311bz44ovMnDkTgLlz5/LNb36Tbdu2MXnyZGpqajratSN/Vpe/oS/wRxCBpJ4FI9LXlJSUUFdXl3Hbvn37KCsro7CwkDVr1vDWW29128+dMGECmzdvZsOGDYwbN45f/OIXXHDBBdTX19PY2Mhll13G9OnTGTduHAAbN25k2rRpTJs2jd/85jds27btiJFMRykgsuGPICylEYRIX1NeXs6MGTM4/fTTKSgooKKi4uC2mTNnMnfuXM444wxOOeUUpk+f3m0/NxaL8bOf/YzPfvazB09S33777ezZs4dZs2bR3NyMc47vf//7ANx3332sX78e5xwXXXQRZ555ZpdrsPaGMSeiXL1Rjs2vw88v4x9bH+Sxb9zb/d8vIu167733OPXUU/NdRq+Q6XdpZkucc1Mz7a9zENk4cIipVSMIEek7dIgpG/4hpmBrC62tjkBADw4Tka658847ef311w9bd/fdd3PLLbfkqaIjKSCy4Y8gYrTQkmolFgjmuSAROdE99NBD+S7hmHSIKRv+CCJqCeJ67aiI9BEKiGz4I4goCb2XWkT6DAVENg6MIEgQTx79AV0iIr2FAiIbGkGISB+kgMhGMIzDiFqL3kstIkdVXFzc7rbNmzdz+umn92A1XaOAyIYZrcGod4gpoYAQkb5Bl7lm6UBAaAQhkke/nwMfruje7xwyCS79drub77//fkaNGsUdd9wBwL/+679iZrz66qvs3buXRCLBN7/5TWbNmtWhH9vc3MwXv/hFFi9eTCgU4nvf+x4XXnghq1at4pZbbqGlpYXW1laef/55hg0bxjXXXENVVRWpVIoHH3yQa6+9tkvdzoYCIksuGNM5CJE+aPbs2Xz5y18+GBDPPPMMf/jDH7jnnnsoLS1l9+7dTJ8+nSuvvBKz7G+iPXAfxIoVK1izZg0XX3wx69atY+7cudx9991cd911tLS0kEqlWLBgAcOGDeN3v/sd4D0ksCcoILLkQlH/PghdxSSSN0f5l36unHXWWezatYsdO3ZQXV1NWVkZQ4cO5Z577uHVV18lEAiwfft2du7cyZAhQ7L+3tdee41/+qd/Arwnt44aNYp169Zx3nnn8a1vfYuqqio+/elPM378eCZNmsS9997L/fffzxVXXMHHPvaxXHX3MDoHka1QjBgtNOschEif85nPfIbnnnuOp59+mtmzZ/Pkk09SXV3NkiVLWLZsGRUVFRnfA3E07T0o9fOf/zzz58+noKCASy65hD//+c+cfPLJLFmyhEmTJvHAAw/wjW98ozu6dUw5DQgzm2lma81sg5nNybD9OjNb7n/eMLMz07ZtNrMVZrbMzHLwiNYOCnsB0dSiEYRIXzN79myeeuopnnvuOT7zmc+wb98+Bg8eTDgc5i9/+Qtbtmzp8Heef/75PPnkkwCsW7eOrVu3csopp7Bp0ybGjh3Ll770Ja688kqWL1/Ojh07KCws5Prrr+fee+9l6dKl3d3FjHJ2iMnMgsBDwCeBKmCRmc13zq1O2+194ALn3F4zuxR4FJiWtv1C59zuXNXYERYuosj2siOhgBDpa0477TTq6uqorKxk6NChXHfddXzqU59i6tSpTJ48mQkTJnT4O++44w5uv/12Jk2aRCgU4uc//znRaJSnn36aJ554gnA4zJAhQ/ja177GokWLuO+++wgEAoTDYR555JEc9PJIOXsfhJmdB/yrc+4Sf/kBAOfc/25n/zJgpXOu0l/eDEztSEDk7H0QQOIXV/Pe+o2888kXuPVjY3PyM0TkSHofRPc5nt4HUQlsS1uu8te15wvA79OWHfCSmS0xs9tyUF+HBCPFFBLXISYR6TNyeRVTpuu9Mg5XzOxCvID4aNrqGc65HWY2GPijma1xzr2aoe1twG0AI0eO7HrV7QhEiyiyZpp0iElEjmHFihXccMMNh62LRqO8/fbbeaqoc3IZEFXAiLTl4cCOtjuZ2RnAT4BLnXM1B9Y753b4011mNg84FzgiIJxzj+Kdu2Dq1Km5e39qpMgbQSggRHqcc65D9xjk26RJk1i2bFm+yzhMZ04n5PIQ0yJgvJmNMbMIMBuYn76DmY0EXgBucM6tS1tfZGYlB+aBi4GVOaz12MKFFFicZgWESI+KxWLU1NR06i848TjnqKmpIRaLdahdzkYQzrmkmd0FLASCwGPOuVVmdru/fS7wNaAceNj/10HSP1lSAczz14WAXzrn/pCrWrMSKSJCknhc76UW6UnDhw+nqqqK6urqfJdyQovFYgwfPrxDbXJ6J7VzbgGwoM26uWnztwK3Zmi3CTiz7fq8ChcC0BpvyHMhIn1LOBxmzJgx+S6jT9Kd1NmKFAHgWhrzXIiISM9QQGTLDwha6vNbh4hID1FAZMs/xGQJjSBEpG9QQGTrwAhCASEifYQCIlt+QAQTOkktIn2DAiJb0VIAQkmdgxCRvkEBka1YPwAiCggR6SMUENmKeSOIotZ6vVVORPoEBUS2woW0WogSa6S+OZnvakREck4BkS0zEuESSmmkIa4RhIj0fgqIDkhGSim1RuriiXyXIiKScwqIDmiNlFJKgw4xiUifoIDoiFg/Sq2R+rgCQkR6PwVEB1hBKaUoIESkb1BAdECwoD8l1kidDjGJSB+ggOiAUFGZRhAi0mfk9IVBvU2osD9hi9PY2JTvUkREck4jiA6wWH8Ako21ea1DRKQnKCA6wn8eU0oBISJ9gAKiI/yAsOba/NYhItIDFBAdUTQQgFDznjwXIiKSewqIjvADIhKvyXMhIiK5p4DoiKJBAETjGkGISO+ngOiISBHxQAEFCQWEiPR+CogOag6XUZLaSzLVmu9SRERySgHRQS2xcsrZT22THvktIr2bAqKDUgUDGWj72dvQku9SRERySgHRUUWDKLd97G3UCEJEerecBoSZzTSztWa2wczmZNh+nZkt9z9vmNmZ2bbNl2DJYAZQx5765nyXIiKSUzkLCDMLAg8BlwITgc+Z2cQ2u70PXOCcOwP4N+DRDrTNi0i/CsKWomHf7nyXIiKSU7kcQZwLbHDObXLOtQBPAbPSd3DOveGc2+svvgUMz7ZtvhQMGApAy97tea5ERCS3chkQlcC2tOUqf117vgD8vqNtzew2M1tsZourq6u7UG52ImUjvZ+7XwEhIr1bLgPCMqxzGXc0uxAvIO7vaFvn3KPOuanOuamDBg3qVKEd0s/LqWD9jtz/LBGRPMrlC4OqgBFpy8OBI/5WNbMzgJ8AlzrnajrSNi+Kh5AiQKzhg3xXIiKSU7kcQSwCxpvZGDOLALOB+ek7mNlI4AXgBufcuo60zZtgiNpgOYXxnfmuREQkp3I2gnDOJc3sLmAhEAQec86tMrPb/e1zga8B5cDDZgaQ9A8XZWybq1o7qj5aQWmTAkJEerecvpPaObcAWNBm3dy0+VuBW7Nte7xoLhjCwPoVJFOthIK611BEeif97dYJqdJhDLU97K6L57sUEZGcUUB0QqD/KGKWYPfOrfkuRUQkZxQQnRCrGA9Awwfr81yJiEjuKCA6oaTyFABSuzfkuRIRkdxRQHRC/6EnkXBBbO/7+S5FRCRnFBCdEAyF+cAGU1C3Jd+liIjkjAKik3ZHKunXpJPUItJ7KSA6qa5oFBWJ7dCqd1OLSO+kgOiklgGnUEgziT06zCQivZMCopNsyGkA7H3/73muREQkNxQQnVQ6chIAjduW57kSEZHcUEB0UmXFYLa2DsJ2rc53KSIiOaGA6KQhpTHWMoqi2rX5LkVEJCcUEJ0UDBg7Y2Mpa94KieZ8lyMi0u0UEF3QNGACQVpBh5lEpBdSQHTF8HMASGxdlOdCRES6nwKiC4YMP4mdrj+Nm97KdykiIt1OAdEF44eU8PfW8QR3LM53KSIi3U4B0QVjBhaxzI2juGErNNTkuxwRkW6lgOiCaCjIhyWnewtVOg8hIr2LAqKLUkPOIkEItrye71JERLqVAqKLJoysYGnrOFIbX8l3KSIi3UoB0UVnDO/H66nTCexcAY178l2OiEi3UUB00RmV/XmjdSKGg82v5bscEZFuk1VAmNndZlZqnp+a2VIzuzjXxZ0I+hWG2Vs2iWaLwaa/5LscEZFuk+0I4h+dc/uBi4FBwC3At3NW1Qlm4ohBvG1nwLqF4Fy+yxER6RbZBoT508uAnznn3k1b1+dNHtGf38TPgv3b4YN3812OiEi3yDYglpjZS3gBsdDMSoBjvozZzGaa2Voz22BmczJsn2Bmb5pZ3MzubbNts5mtMLNlZnZc36o8bcwA/pw6C0cA1i7IdzkiIt0i24D4AjAHOMc51wiE8Q4ztcvMgsBDwKXAROBzZjaxzW57gC8B32nnay50zk12zk3Nss68OHVoKYnYADYXnq6AEJFeI9uAOA9Y65yrNbPrga8C+47R5lxgg3Nuk3OuBXgKmJW+g3Nul3NuEZDoYN3HlWDAOHf0ABYmp8CHK2Dv5nyXJCLSZdkGxCNAo5mdCXwF2AI8fow2lcC2tOUqf122HPCSmS0xs9va28nMbjOzxWa2uLq6ugNf372mjR3AL/af5S0sfzZvdYiIdJdsAyLpnHN4I4D/cM79B1ByjDaZTmJ35BKfGc65KXiHqO40s/Mz7eSce9Q5N9U5N3XQoEEd+Pru9ZGTBrKdQVSXnwPLn9bVTCJywss2IOrM7AHgBuB3/vmF8DHaVAEj0paHAzuyLcw5t8Of7gLm4R2yOm6dNqyUitIoC4PnQ8162LE03yWJiHRJtgFxLRDHux/iQ7xDRf/3GG0WAePNbIyZRYDZwPxsfpiZFflXSmFmRXj3X6zMsta8MDMuPGUwD+88HReMwrtP5bskEZEuySog/FB4EuhnZlcAzc65o56DcM4lgbuAhcB7wDPOuVVmdruZ3Q5gZkPMrAr4Z+CrZlZlZqVABfCamb0LvAP8zjn3h072scd8YsJgdsSj7B5xiRcQ8fp8lyQi0mmhbHYys2vwRgyv4J1b+E8zu88599zR2jnnFgAL2qybmzb/Id6hp7b2A2dmU9vxZMa4gUSCAX4bu4Jb4vO9cxHnfCHfZYmIdEq2h5j+Be8eiJucczfinQ94MHdlnZiKoiE+Mq6cn24eiBtyBrzzY52sFpETVrYBEfBPFh9Q04G2fcqnzhhGVW0zW066Hqrf0xNeReSEle1f8n8ws4VmdrOZ3Qz8jjaHjsRz8WkVREMBnmg4BwrL4fX/yHdJIiKdku1J6vuAR4Ez8M4NPOqcuz+XhZ2oSmJhLjp1MC+urCE17Q7Y8EfYrkteReTEk/VhIufc8865f3bO3eOcm5fLok50V545jN31Lbxe/g8Q6w+vHuuKYBGR489RA8LM6sxsf4ZPnZnt76kiTzSfmFBBeVGEJ5buhel3eA/w02PAReQEc9SAcM6VOOdKM3xKnHOlPVXkiSYSCnDNOSN4ec0uPjz1Jigog5ce1BVNInJC0ZVIOfK5c0bS6hxPrdgPH38A3v8rrH8p32WJiGRNAZEjI8sLOX/8IJ54ayvNZ94E5ePgpa9C6oR+srmI9CEKiBz67+ePZXd9nBfe3QUXfxN2r4M3/jPfZYmIZEUBkUPnnVTOmcP78aNXN5IaPxNOvRJe+TbUbMx3aSIix6SAyCEz44sfP4ktNY0sWPEBXPZ/IRSD+V+C1mO+0ltEJK8UEDl28cQhjBtczA/+tI5k4WC4+N9gy2vwzo/yXZqIyFEpIHIsEDDuvfgUNlY38OySKphyI5x8Kfzxa7BjWb7LExFplwKiB1xyWgVTRvbn+39cR1OiFa56GAoHwnO3QLwu3+WJiGSkgOgBZsYDl53Krro4c/+6EQoHwNU/gb2bYd7tOh8hIsclBUQPOWf0AD515jAeeWUj7+9ugNEz4OJvwZrfwl++me/yRESOoIDoQQ9efirRUIAHX1yJcw6mf9E7J/G378KyX+W7PBGRwyggetDg0hhfmXkKr23YzbOLq8AMLvsujLkAfn0nvPfbfJcoInKQAqKHfX7aKKaPHcDXf7OKLTUNEIrA7F9C5RTvpPXGP+e7RBERQAHR44IB47vXTCYQMO55ehnJVCtEi+G6Z2HgyfCrz8M6PdRPRPJPAZEHlf0L+NY/TGLp1lq++8d13sqCMrjx1zBwPDz1OVj5fH6LFJE+TwGRJ1eeOYzPnTuSR17ZyG+X7/BWFg2Em38LI6bBc1+ANx/SOyREJG8UEHn09StPY+qoMu57djmrd/gv6Iv1g+ufh1OvgIX/wzt5nYznt1AR6ZMUEHkUCQV4+Pop9CsIc/PP3mHbnkZvQ7gAPvs4XDAHlj0JP78carflt1gR6XMUEHk2uCTG4184l3iylet/+jbVdf5oIRCACx+Aax6HXe/B3Bmwal5+ixWRPkUBcRw4uaKEx24+h13749z42DvsaWg5tHHiLLj9b1A+Hp69GV68E5pq81WqiPQhOQ0IM5tpZmvNbIOZzcmwfYKZvWlmcTO7tyNte5uzR5XxoxvOZlN1Pdf+6E127m8+tHHAWPjHP8D598G7v4SHzvVGEzqBLSI5lLOAMLMg8BBwKTAR+JyZTWyz2x7gS8B3OtG21zn/5EH8/JZz2VHbxDU/epOtNY2HNgbD8Imvwq0vQ8kQbzTx5Gdh9/q81SsivVsuRxDnAhucc5uccy3AU8Cs9B2cc7ucc4uAREfb9lbnnVTOE7dOo7YxwVUPv8477+85fIfKKXDrn2Hmt2Hrm/DQNPjd/wf11fkpWER6rVwGRCWQfulNlb+uW9ua2W1mttjMFldX946/JM8aWcaLd86gf0GY637yFs8sbnMFUzDkPejvS3+Hs2+GxT+DH06Gv/wvaNyT6StFRDoslwFhGdZle9A867bOuUedc1Odc1MHDRqUdXHHuzEDi5h3xwymjSnnK88t5yvPvUtjS/LwnYoHwxXfgzvfhpMuhL/+O3z/dHjpq1C3Mz+Fi0ivkcuAqAJGpC0PB3b0QNteo19hmJ/fcg53XTiOZ5dUccV/vsbK7fuO3HHgeLj2CfjimzDhMu8O7B9MghfvgO1Le75wEekVchkQi4DxZjbGzCLAbGB+D7TtVULBAPdecgpP3jqNhniSqx56ne8sXEtzInXkzhUTvTfV3bUYzroeVr0IP74QfnwRvPsUtDQe2UZEpB3mcnippJldBvwACAKPOee+ZWa3Azjn5prZEGAxUAq0AvXAROfc/kxtj/Xzpk6d6hYvXpyTvhwP9ja08G+/W80LS7czdmAR3/qHSZx3Unn7DZr3ecHwzqNQswEiJd59FWfOhlEzvJvxRKRPM7MlzrmpGbflMiB6Wm8PiAP+tr6a/zFvBdv2NHHJaRXcP3MCYwcVt9+gtRW2vO6FxepfQ0sd9BsBp18Np17pXRllmU77iEhvp4DohZpaUvz0tU088spG4slWrps2kjs/MY7BJbGjN2xphLUL4N1fwaZXoDUJJcO8hwNOuMIbWQRDPdIHEck/BUQvVl0X5wd/WsdTi7YRDBjXTh3Bf79gLMPLCo/duGkvrFsI7/0GNrwMySbvabJjPw4nfQJOugj6jzjm14jIiUsB0Qds3t3A3L9u5PmlVTgHsyZXctNHRnHG8P7ZfUFLgxcS6xfChj9DnX/RWPl4LyzGXgAjz4PCATnrg4j0PAVEH7KjtolHX93EM4u30diS4swR/blx+iguP2MosXAwuy9xDqrXeu/H3vgybH7dG10ADJrgBcWoj3hTjTBETmgKiD5of3OCeUu38/ibm9lY3UBpLMTlZwzj01MqOXtkGYFAB05KJ5phx1LY8ob3eI+tb3snusE72V05BYZNgWFnwbDJ3mEqETkhKCD6MOccb26s4ZnF21i4aidNiRTDywq4anIlM08fwmnDSrGOXsHUmoKdK2HLm7DtLe9mvNoth7aXjz8UGkMmefdnFJR1b8dEpFsoIASAhniSl1Z/yAtLt/P6ht20OhjWL8YnJ1Zw8WlDOHfMAMLBTt4b0VADH/wdtv/dG21sXwr1Hx7aXloJFafB4IlQcboXGuXjIRTpns6JSKcoIOQINfVxXl6ziz+u3snf1lfTnGilJBpi+knlfGz8QGaMG8jYgUUdH12k2/8B7FzljTZ2roJdq71zG63+w3sDYSgfBwPHwcCTvcAYON5bV9C/W/opIkengJCjampJ8bf11fxl7S7+tn43VXu9E9LD+sWYMW4g08aWc/aoMkaXF3YtMACSLd5d3QeCY/c6750We9/37sk4oGiwFxoDx3nBMWAM9B8FZaMgWtK1GkTkIAWEdMjWmkZe27Cb1zZU88bGGmobvX/xlxdFmDKqjLNHlTF1VBmnV/bL/sqoY0klYO/mQ4Gxez3UrPeWm/Yevm/hQCgb7X9Gpc2P9g5lBbqpJpE+QAEhndba6thYXc/iLXtZsmUvS7fsZdPuBgCCAWPcoGJOG1bKaZX9OG1YKROHlVIaC3dvEY17vPBI/9Ru8afbwKU9uNCCUDIU+lV6YdGvEkqHpy0Ph6JBerSIiE8BId2qpj7O0q21rKiqZdWO/azcsY+d++MHt48uL2TCkFLGVxQzbnAxJ1eUMGZgUfeNNtKlkrC/Ki04tsH+7bCvyp9uh1T88DbBCJQO8y7RLR0GxRXep2TI4dNoiYJEej0FhOTcrrpmVu3Yz+od+1m5fR9rd9axpaaRVKv35ytgMKq8iPGDixlfUcyYgcWMLi9kVHkRA4sjXT+30R7noLEG9m3zwqJteOzfAfU7jwwRgHCh91Km4iFQUpE2rfDOkRQN9D6FAyGSxaNNRI5DCgjJi3gyxfu7G1i/s571u+rZsKuO9TvreX93A8nWQ3/uiiJBRpYXHQyMUeWFjCovZERZIUP6xTp/6W22nIPmWu8tfPUfQv0uqPvQC44D0/qd3vZ4hhc2gRcmhQOhqNyfDoTC8kMBUjTo8HWRYo1O5LhwtIDQYzslZ6KhIBOGlDJhSOlh6xOpVqr2NrGlpoEtNY1srmlga00j63bW8fJ7u2hJtR7cN2AwuCRGZVkBw/oXMKx/jMr+BQzr5y1X9i+gtCDUtRGImXcjX0EZDJ5w9H1bGr2waNgNjbu9aUO1N0o5uK4aqtd4ywceUdJWIHzoZxaUeZf1Hrbsf2L9D98e66eT8NJjFBDS48LBAGMGFjFmYNER21Ktjg/2NbGlppHte5vYXut9dtQ2saKqloUrmw8LEPBGIBWlMQaVRBlcGqOiJMrg0iiDS2KHTUuiXQwS8A4lDRjjfbLR0uAFRkPNoUBp3O1dmZX+2b8ddq725g88xiQjg1jpoQCJlnrL0X7+tLSdadr28DEeCS/iU0DIcSUYMIaXFbb7uPLWVkdNQws7/NA4ECC76uLs2t/M8qpadu2P05ThlayxcICK0hiDS6IMLI4yoChCeVGEAUURBhRHD86XF0UoK4p0z6GtSJH3KRudfZtUAppqvbBorm0TJm2Wm/d5I5rm/RDfDy31x/7+YCRzkByYjxR5h8AixRAtPnw5UuSv8z+hqA6V9WIKCDmhBALGoJIog0qinDmif8Z9nHPUx5Ps3B9nV10z1XVxdu2Ps3N/sxckdc1s2FXPnoYW9jS20N5puH4F4UMBUhShvDhCWWGE/oVh+hWE6VdwaP7AtCAc7PooJRiG4kHep6NaUxCv88LiQGgcnO5rZ/1+aNjkL9d5IeMyvPM8EwseHhiHBUhR2roSbxou9D4RfxouOLQuXODvUwChAr0S9ziggJBex8woiYUpiYUZN/gor2LFO6RV29jCnoYWahrSpvUt1DTED85vqWlk6dZa9ja2HLwyK5NIMEC/A6HhB0dpQZj+BZGDQVJaEKI4GqYkFqI4GqI05s/HQl0ftQSC/vmK/p3/DucgGfeCoqUe4vXeobKWOm+azXLjtsOX2zsXczShgkMBEskQJscKmlDU+45Q1A+dKIRihz5hfxqMaBTUDgWE9GnBgFFeHKW8OMr4LPZ3ztHQkqK2sYV9TQn2NSaobUqwrylBbaM33dfUcnB5R20z731Qx76mBPXx5DG/PxYOUBwNU+oHRkksREk0nDYfoiR2aLk46n0KIyGKosGD0y6NZMy8vzzDMe+Kq+7QmvLCJtEEiUbvZP+B+UQTJBr8aZMXKodtazw039LgjYTqPjxyfbajniM77IdGepCkBUo4ljlYjhpAUQhGvYdRHjaNeoF02DR63I6WFBAiHWBmB/9SHt7BJ5gnUq3sb0qwvzlJfXOSumZ/Pu7N1zcnqfPn65qT1PnbquvqvXl/e3Z1QmE4SGE0RFEkeESAFEb89QcDJkhRJERh1J/6bQoiQQoiQWKhgD8NduxdIgcEgt4VWLl8V0iy5fBQSTZ77zJJNnsjmGTc25aM++ua2+zTdjnutUs0eXfzH1hOxg/tk+n+mc4IhDoQKJEjA6igDC58oHtqSaOAEOkh4WDg4Gils1pbHfUtBwImSX08QUM8RWNL8tC0JUVj3J+mr497I5/ttYe2N8STh92Tko2oHxYFYe8TDQcpCB9aFwunTSOHlg/sE0vbJ71NNBQgGg4QDfnzoUDHRkGhiPfpyScBt7Z6IZEpaJIt/rYD0zikWtpM4+3slzhyXbLZGz1l+g4FhIgEAkZpLNytz7tqSbYeGSz+tCmRotn/NPnLTYkUzQfnWw/bXtuYaLM9RXOi9dhFtCMSChALBYiGD4VGNBT0gyQtTI653ZuPtQmg9HaRUIBI0JuG/WkoYEcPqUAAAv65koJOd/O4pYAQ6eMioQCRUIT+OXpaiHOOeLL1sIBpavFDJS144slW75M+n0wRT6TNJ1v9ZW++tilBPJGixd+/+WDbFIlU158SYeaN/KLBAGE/QMIh86ZBL1jCbULlYNAc3DdIOGTed7TdN63NoW125Pf6bcNBI3Rw2QgeK8C6SAEhIjllZgcPK/Xki2dTrc4PjiODpW3wNCdaaUm20pLypom0aTzVSiLpaEml/GnmfRviSeIHlg+2aSWR9L6jJdn5kdTRRIIBKvpF+dtXPtHt362AEJFeKRiwgyfZjwfOOZJ+aCWOCBmXMXRaUt58IuX8qbc+2epIHAwjRyycm6ugFBAiIj3AzAgHLfcPn+xGOa3UzGaa2Voz22BmczJsNzP7ob99uZlNSdu22cxWmNkyM9MjWkVEeljORhBmFgQeAj4JVAGLzGy+c2512m6XAuP9zzTgEX96wIXOud25qlFERNqXyxHEucAG59wm51wL8BQwq80+s4DHnectoL+ZDc1hTSIikqVcBkQlsC1tucpfl+0+DnjJzJaY2W05q1JERDLK5UnqTBfntr0w+Wj7zHDO7TCzwcAfzWyNc+7VI36IFx63AYwcObIr9YqISJpcjiCqgBFpy8OBHdnu45w7MN0FzMM7ZHUE59yjzrmpzrmpgwZ14vHIIiKSUS4DYhEw3szGmFkEmA3Mb7PPfOBG/2qm6cA+59wHZlZkZiUAZlYEXAyszGGtIiLSRs4OMTnnkmZ2F7AQCAKPOedWmdnt/va5wALgMmAD0Ajc4jevAOb5t5CHgF865/6Qq1pFRORI5tp7ndYJaOrUqW7xYt0yISKSLTNb4pybmmnbiXNLn4iI9CgFhIiIZKSAEBGRjBQQIiKSkQJCREQyUkCIiEhGCggREclIASEiIhkpIEREJCMFhIiIZKSAEBGRjBQQIiKSkQJCREQyUkCIiEhGCggREclIASEiIhkpIEREJCMFhIiIZKSAEBGRjBQQIiKSkQJCREQyUkCIiEhGCggREclIASEiIhkpIEREJCMFhIiIZKSAEBGRjBQQIiKSUU4DwsxmmtlaM9tgZnMybDcz+6G/fbmZTcm2rYiI5FbOAsLMgsBDwKXAROBzZjaxzW6XAuP9z23AIx1oKyIiOZTLEcS5wAbn3CbnXAvwFDCrzT6zgMed5y2gv5kNzbKtiIjkUCiH310JbEtbrgKmZbFPZZZtATCz2/BGHwD1Zra2k/UOBHZ3su2JSn3uG9Tn3q8r/R3V3oZcBoRlWOey3Cebtt5K5x4FHu1YaUcys8XOuald/Z4TifrcN6jPvV+u+pvLgKgCRqQtDwd2ZLlPJIu2IiKSQ7k8B7EIGG9mY8wsAswG5rfZZz5wo38103Rgn3PugyzbiohIDuVsBOGcS5rZXcBCIAg85pxbZWa3+9vnAguAy4ANQCNwy9Ha5qpWX5cPU52A1Oe+QX3u/XLSX3Mu46F9ERHp43QntYiIZKSAEBGRjPp8QPTWR3qY2Qgz+4uZvWdmq8zsbn/9ADP7o5mt96dlaW0e8H8Pa83skvxV3zVmFjSzv5vZb/3lXt1nM+tvZs+Z2Rr/v/d5faDP9/h/rlea2a/MLNbb+mxmj5nZLjNbmbauw300s7PNbIW/7Ydmluk2gsycc332g3cCfCMwFu/S2neBifmuq5v6NhSY4s+XAOvwHlvyf4A5/vo5wL/78xP9/keBMf7vJZjvfnSy7/8M/BL4rb/cq/sM/Bdwqz8fAfr35j7j3Uj7PlDgLz8D3Nzb+gycD0wBVqat63AfgXeA8/DuL/s9cGm2NfT1EUSvfaSHc+4D59xSf74OeA/vf6xZeH+h4E+v8udnAU855+LOuffxriw7t0eL7gZmNhy4HPhJ2upe22czK8X7i+SnAM65FudcLb24z74QUGBmIaAQ7z6pXtVn59yrwJ42qzvUR//RRaXOuTedlxaPp7U5pr4eEO096qNXMbPRwFnA20CF8+41wZ8O9nfrLb+LHwBfAVrT1vXmPo8FqoGf+YfVfmJmRfTiPjvntgPfAbYCH+DdP/USvbjPaTrax0p/vu36rPT1gMj6kR4nKjMrBp4Hvuyc23+0XTOsO6F+F2Z2BbDLObck2yYZ1p1Qfcb7l/QU4BHn3FlAA96hh/ac8H32j7vPwjuUMgwoMrPrj9Ykw7oTqs9Z6PJjizLp6wGRzeNATlhmFsYLhyedcy/4q3f6w0786S5/fW/4XcwArjSzzXiHCz9hZk/Qu/tcBVQ55972l5/DC4ze3Of/BrzvnKt2ziWAF4CP0Lv7fEBH+1jlz7ddn5W+HhC99pEe/pUKPwXec859L23TfOAmf/4m4Ndp62ebWdTMxuC9o+Odnqq3OzjnHnDODXfOjcb7b/ln59z19O4+fwhsM7NT/FUXAavpxX3GO7Q03cwK/T/nF+GdY+vNfT6gQ330D0PVmdl0/3d1Y1qbY8v3mfp8f/Ae9bEO76z/v+S7nm7s10fxhpLLgWX+5zKgHHgZWO9PB6S1+Rf/97CWDlzpcDx+gI9z6CqmXt1nYDKw2P9v/SJQ1gf6/HVgDbAS+AXe1Tu9qs/Ar/DOsSTwRgJf6Ewfgan+72kj8P/jP0Ejm48etSEiIhn19UNMIiLSDgWEiIhkpIAQEZGMFBAiIpKRAkJERDJSQIgcB8zs4weePityvFBAiIhIRgoIkQ4ws+vN7B0zW2ZmP/LfPVFvZt81s6Vm9rKZDfL3nWxmb5nZcjObd+DZ/WY2zsz+ZGbv+m1O8r++OO29Dk926Ln9IjmggBDJkpmdClwLzHDOTQZSwHVAEbDUOTcF+CvwP/0mjwP3O+fOAFakrX8SeMg5dybeM4Q+8NefBXwZ79n+Y/GeLSWSN6F8FyByArkIOBtY5P/jvgDvYWmtwNP+Pk8AL5hZP6C/c+6v/vr/Ap41sxKg0jk3D8A51wzgf987zrkqf3kZMBp4Lee9EmmHAkIkewb8l3PugcNWmj3YZr+jPb/maIeN4mnzKfT/p+SZDjGJZO9l4DNmNhgOvh94FN7/R5/x9/k88Jpzbh+w18w+5q+/Afir897JUWVmV/nfETWzwp7shEi29C8UkSw551ab2VeBl8wsgPeUzTvxXtJzmpktAfbhnacA73HMc/0A2ATc4q+/AfiRmX3D/47P9mA3RLKmp7mKdJGZ1TvnivNdh0h30yEmERHJSCMIERHJSCMIERHJSAEhIiIZKSBERCQjBYSIiGSkgBARkYz+HwtEdxZEUKZ7AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.ylim(0, 0.3)\n",
    "plt.plot(single_layer.losses)\n",
    "plt.plot(single_layer.val_losses)\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train_loss', 'val_loss'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEGCAYAAAB7DNKzAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAj5ElEQVR4nO3dd3yV9d3/8deHvZdsQgwgG0QhglatKKKAIu46a7ltkf601fbXOoqrVgtaW4U6uFPEVWerCCq4qAMVlSFKSNgz7D3CCEk+9x/neN/HmD3Odc7J+/l45MEZX3LekJy8c13n+lzH3B0REZGi1Ag6gIiIxDYVhYiIFEtFISIixVJRiIhIsVQUIiJSrFpBB6gKLVu29JSUlKBjiIjEjQULFuxw91aF3ZeQRZGSksL8+fODjiEiEjfMbF1R92nXk4iIFEtFISIixVJRiIhIsVQUIiJSLBWFiIgUK9CiMLOpZrbNzNKLuN/MbJKZrTSzb82sf7QziohUd0FvUTwDDCvm/uFA1/DHGODJKGQSEZEIgRaFu38C7CpmySjgOQ/5AmhmZu2ik05EJH58uXonkz9eVSWfO+gtipJ0ADZEXM8K3/YDZjbGzOab2fzt27dHJZyISNB2HjjC7/71DT9J+4IXv1zPwZzcSn+MWJ/MtkJuK/Sdltw9DUgDSE1N1bsxiUhCy893Xp2/gfGzlpJ9JJdfDu7Cr8/qSv06NSv9sWK9KLKAjhHXk4BNAWUREYkJmZv3MW7aYhau38PATi24/8I+dGvTuMoeL9aLYgZwk5m9DAwC9rr75oAziYgEIvtILhNnr+CpT9fQtH5tHr6sH5f074BZYTtfKk+gRWFmLwGDgZZmlgXcA9QGcPfJwExgBLASOAiMDiapiEhw3J33MrbyxxlL2LT3MFcO7Mit5/agecM6UXn8QIvC3a8s4X4HboxSHBGRmLNh10HunbGE2Uu30aNtY/5+1YkMOLZFVDPE+q4nEZFqKSc3n6c+XcPE2cupYca4ET352akp1K4Z/YNVVRQiIjHmy9U7ufONdFZsO8C5vdtwz8jetG9WP7A8KgoRkRix88ARxs9ayr8XZNGhWX2eui6VIT3bBB1LRSEiErTvZiImvLOUA4erdiaiPFQUIiIBWrplH+OmpbNg3e6ozESUh4pCRCQAQc1ElIeKQkQkigrORFxxUkduGxa9mYjyUFGIiERJwZmISVeeSGpKdGciykNFISJSxWJpJqI8VBQiIlUocibinF5tuOeC3nQIcCaiPFQUIiJVoOBMxJSfpnJ2r+BnIspDRSEiUokKm4n41VnH0aBO/P64jd/kIiIx5nszESktuP+i2JuJKA8VhYhIBRWcifjLpcdz6YCkmJyJKA8VhYhIBby3ZAv3xtFMRHmoKEREyiFrd2gm4oPM+JqJKA8VhYhIGRzNy2fKnDVMmr0CM+JuJqI8VBQiIqX01Zpd3PnGYpZvjd+ZiPII+j2zhwETgZrAFHefUOD+psA/gWRCWR9296ejHlREqrVd2TmMn5nJvxJgJqI8AisKM6sJPA4MBbKAeWY2w90zIpbdCGS4+0gzawUsM7MX3D0ngMgiUs3k5zv/WrCB8bNCMxFjz+jCr4fE90xEeQT5rx0IrHT31QBm9jIwCogsCgcaW+gYs0bALiA32kFFpPpJ1JmI8giyKDoAGyKuZwGDCqx5DJgBbAIaAz9x9/zCPpmZjQHGACQnJ1d6WBGpHiJnIprUq5VwMxHlEWRRFPa/7gWunwssAs4CugDvm9kcd9/3g7/ongakAaSmphb8PCIiJaoOMxHlEWRRZAEdI64nEdpyiDQamODuDqw0szVAD+Cr6EQUkeogciaie5vG/DuBZyLKI8iimAd0NbNOwEbgCuCqAmvWA0OAOWbWBugOrI5qShFJWJEzEQB/GNGD0ad2SuiZiPIIrCjcPdfMbgLeJXR47FR3X2JmY8P3Twb+BDxjZosJ7aq6zd13BJVZRBJHdZ2JKI9Aj/Fy95nAzAK3TY64vAk4J9q5RCRxVfeZiPKoXgcDi0i1pZmI8tP/kIgkvKVb9nHntHTmayaiXFQUIpKwDubkMvGD0ExEY81ElJuKQkQS0ntLtvDHNzPYuOeQZiIqSEUhIgklNBORwQeZW0MzEWNP0UxEBakoRCQhHM3L56lP1zDxg9BMxB3De/Bfp2kmojKoKEQk7kXORAzt1YZ7NRNRqVQUIhK3Cs5E/OOnqQzVTESlU1GISNzRTER06X9VROJK5EzESSnNuf/CvnRvq5mIqqSiEJG4UHAm4qFLj+fS/knUqKGZiKqmohCRmBc5E/GT1I7cPlwzEdGkohCRmKWZiNigohCRmKOZiNiiohCRmKKZiNijohCRmKCZiNilohCRQOXnO/9ekMWfZ2VqJiJGBfqVMLNhwERCb4U6xd0nFLJmMPAoUBvY4e5nRDGiiFShZVv2c+cbi5m3VjMRsSywojCzmsDjwFAgC5hnZjPcPSNiTTPgCWCYu683s9aBhBWRSnUwJ5eJs1fw1JzwTMQlofeJ0ExEbApyi2IgsNLdVwOY2cvAKCAjYs1VwOvuvh7A3bdFPaWIVKr3M7Zy74wlbNxziMtTk7h9eE9aaCYipgVZFB2ADRHXs4BBBdZ0A2qb2UdAY2Ciuz8XnXgiUpkiZyK6tWnEv8aewkmaiYgLQRZFYduYXuB6LWAAMASoD8w1sy/cffkPPpnZGGAMQHJyciVHFZHy0kxE/AuyKLKAjhHXk4BNhazZ4e7ZQLaZfQL0A35QFO6eBqQBpKamFiwcEQnAvLW7uHNaOsu27mdorzbcM7IXSc0bBB1LyijIopgHdDWzTsBG4ApCr0lEmg48Zma1gDqEdk09EtWUIlJmu7JzmDArk1fnayYiEQRWFO6ea2Y3Ae8SOjx2qrsvMbOx4fsnu3ummb0DfAvkEzqENj2ozCJSvO9mIsbPymT/4VxuOKMzNw/pqpmIOGfuibeXJjU11efPnx90DJFqRTMR8c3MFrh7amH3qeZFpEI0E5H4VBQiUm6aiageVBQiUmaaiaheVBQiUmqaiaieVBQiUiqRMxFn92zDvRdoJqK6UFGISLEiZyLaN61H2rUDOKd326BjSRSpKESkUPn5zr8XZjF+Zngm4sed+fWQrjSsqx8b1Y2+4iLyA5EzEanHNuf+i/rQo22ToGNJQFQUIvK/ImciGmkmQsJUFCICaCZCiqaiEKnmNu45xL0zlvB+hmYipHAqCpFq6mhePlM/XcOj4ZmI24f34HrNREghVBQi1dD8tbsYp5kIKSUVhUg1sjs7hwmzlvLK/A2aiZBSU1GIVAOaiZCK0HeJSIJbvnU/d05L56u1uzQTIeWiohBJUAdzcpk0eyVT5qymUb1aPHhJXy4b0FEzEVJmKgqRBDQ7cyt3Tw/NRFw2IIk7RmgmQsov0OPgzGyYmS0zs5Vmdnsx604yszwzuzSa+UTizcY9hxjz3Hyuf3Y+DerU5NUbTuEvl/VTSUiFBLZFYWY1gceBoUAWMM/MZrh7RiHrHgTejX5KkfhwNC+fZz5byyMfLCffnduGhWYi6tTSTIRUXJC7ngYCK919NYCZvQyMAjIKrPsV8BpwUnTjicSHBet2M27aYpZu2c+QHq2594LedGyhmQipPEEWRQdgQ8T1LGBQ5AIz6wBcBJxFCUVhZmOAMQDJycmVGlQkFu05mMOD7yzlpa820K5pPf772gGc06sNZnqxWipXkEVR2HezF7j+KHCbu+eV9M3v7mlAGkBqamrBzyOSMNyd1xdu5IGZmew9dJRfnN6JW87uppkIqTJBfmdlAR0jricBmwqsSQVeDpdES2CEmeW6+xtRSSgSY1Zu28+4ael8uWYX/ZOb8cBFfenZTjMRUrWCLIp5QFcz6wRsBK4Aropc4O6dvrtsZs8Ab6kkpDo6lJPHYx+uIO2T1TSoU4vxF/flJ6maiZDoCKwo3D3XzG4idDRTTWCquy8xs7Hh+ycHlU0klny4dBt3z0hnw65DXNy/A38Y0ZOWjeoGHUuqkUB3arr7TGBmgdsKLQh3/1k0MonEis17D3HfmxnMSt9Cl1YNeekXJ3NKl2OCjiXVkF79EokxuXn5PDt3HX97bxm5+c7vz+3OL07vrJkICYyKQiSGfL1+N+OmpZOxeR+Du7fivgv6kHyMZiIkWCoKkRiw9+BRHnp3KS9+tZ7Wjevy5NX9GdanrWYiJCaoKEQC5O5MX7SJ+9/OYFd2DqN/1InfntONRpqJkBii70aRgKzafoC73kjn81U76dexGc+MHkifDk2DjiXyAyUWhZlNKsXn2efud1ZCHpGEd/hoHk98uJLJH6+mbu0a3H9hH64cmExNzURIjCrNFsUo4O4S1twOqChESvDx8u3cPT2ddTsPcuEJ7Rl3Xi9aNdZMhMS20hTFI+7+bHELzKx5JeURSUhb9x3mvrcyePvbzXRu2ZAXfj6IU49rGXQskVIpsSjc/dHKWCNSHeXlO8/PXcvD7y0nJy+f3w7txg1ndKZurZpBRxMptdK8RmHAZYTO7PpvQqf8HgUsBSa7e36VJhSJU99m7WHctHQWb9zL6V1b8qdRfUhp2TDoWCJlVppdT48DrYE6hAqiLvAmMALoDtxcZelE4tC+w0d5+N1lPP/FOlo1qsvfrzyR849vp5kIiVulKYrT3b2vmdUGtgDt3D3HzF4Evq7aeCLxw92Z8c0m7n87k50HjnDdKSn89pxuNKlXO+hoIhVSmqLIBXD3o2Y2z91zwtdzzSyvStOJxIk1O7K56410Pl25g+OTmjL1upPom6SZCEkMpSmKLWbWyN0PuPuw7240s7ZATtVFE4l9h4/mMfnjVTzx0Srq1qzBfaN6c/WgYzUTIQmlNEc9DS/irv3A+ZUbRyR+fLpiB3dNT2fNjmxG9mvPXef1pHWTekHHEql0pTnqqa27byl4u7tnA9nFrRFJRNv2H+b+tzKZ8c0mUo5pwPPXD+T0rq2CjiVSZUqz62km0L8S1ojEtbx854Uv1/GXd5dx5Gg+Nw/pyi8Hd6Febc1ESGIrTVH0M7N9BW4ruAO24P0iCSV9417+MG0x32bt5bTjWnLfqN50btUo6FgiUVGa1yhqApjZ88AcYI67Z1bGg5vZMGAioffMnuLuEwrcfzVwW/jqAeCX7v5NZTy2SGnsP3yUv763nOfmrqVFw7pMvOIELujXXjMRUq2U5TTjTwOnAZPMrDOhGYo57j6xPA9sZjUJDfMNBbKAeWY2w90zIpatAc5w991mNhxIAwaV5/FEysLdeXvxZu57M4PtB45wzaBj+d253WlaXzMRUv2Uuijc/T9m9jFwEnAmMBboQ2iLoDwGAivdfTWAmb1MaPL7f4vC3T+PWP8FkFTOxxIptXU7s7lr+hI+Wb6d3u2b8I+fptKvY7OgY4kEptRFYWazgYbAXEK7oE5y920VeOwOwIaI61kUv7VwPTCrmHxjgDEAycnJFYgl1dWR3DzSPl7NYx+upHbNGtwzshfXnnwstWrWCDqaSKDKsuvpW2AAoa2IvcAeM5vr7ofK+diF7eT1QheanUmoKE4r6pO5exqhXVOkpqYW+nlEivL5yh3cOT2d1duzOa9vO+46vxdtm2omQgTKtuvpNwBm1ggYTeg1i7aEThJYHllAx4jrScCmgovM7HhgCjDc3XeW87FECrV9/xH+PDOTaV9vJLlFA54ZfRKDu7cOOpZITCnLrqebgNMJbVWsA6YS2gVVXvOArmbWCdgIXAFcVeAxk4HXgWvdfXkFHkvke/LznZfmrefBWUs5dDSPX511HDeeeZxmIkQKUZZdT/WBvwEL3D23og8cPqngTcC7hA6PneruS8xsbPj+yYTegvUY4Inw4Yi57p5a0ceW6m3Jpr2Mm5bOog17OKXzMfzpwj4c11ozESJFMffE252fmprq8+fPDzqGxJgDR3J55P3lPP3ZGpo3qMOd5/fkwhM6aCZCBDCzBUX9Il6WLQqRuOTuvJO+hT++mcHW/Ye5amAyt57bg6YNNBMhUhoqCkloG3Yd5O7p6Xy4bDs92zXhiWv60z+5edCxROKKikISUk5uPv+Ys5q//2cFNc2487ye/OxHKZqJECkHFYUknC9W7+TON9JZue0Aw/u05e6RvWjXtH7QsUTilopCEsbOA0f488ylvLYwi6Tm9Zn6s1TO6tEm6FgicU9FIXEvP995df4Gxs9aysGcXG48sws3ndmV+nU0EyFSGVQUEtcyN+9j3LTFLFy/h4GdWvDAhX3o2qZx0LFEEoqKQuJS9pFcJs5ewVOfrqFp/do8fFk/LumvmQiRqqCikLjz3pIt3DtjCZv2HubKgR259dweNG9YJ+hYIglLRSFxI2v3Qe6dsYQPMrfRo21jJl15IqkpLYKOJZLwVBQS847m5fPUp2uY+MEKAP4wogejT+1Ebc1EiESFikJi2ry1uxg3bTHLtx7gnF5tuOeC3nRoppkIkWhSUUhM2pWdw4RZmbw6P4sOzerzj5+mMrSXZiJEgqCikJiSn+/8e2EW42dmsv9wLmPP6MKvhxxHgzr6VhUJip59EjOWbdnPnW8sZt7a3ZyU0pz7L+xL97aaiRAJmopCAncwJ5dJs1cyZc5qGtWrxUOXHM+lA5KoUUMzESKxQEUhgZqduZW7py9h455DXJ6axO3De9JCMxEiMSXQojCzYcBEQm+FOsXdJxS438L3jwAOAj9z94VRDyqVbuOeQ/xxxhLey9hKtzaNePWGUxjYSTMRIrEosKIws5rA48BQIAuYZ2Yz3D0jYtlwoGv4YxDwZPhPiVNH8/J55rO1PPLBcvLduW1YD64/rRN1amkmQiRWBblFMRBY6e6rAczsZWAUEFkUo4DnPPTG3l+YWTMza+fum6MfVypqwbpdjJuWztIt+xnSozX3XtCbji0aBB1LREoQZFF0ADZEXM/ih1sLha3pAPygKMxsDDAGIDk5uVKDSsXsOZjDg+8s5aWvNtCuaT3++9oBnNOrjU7gJxIngiyKwn5KeDnWhG50TwPSAFJTUwtdI9Hl7ry+cCMPzMxk76GjjPlxZ24e0pWGdXUMhUg8CfIZmwV0jLieBGwqxxqJQSu37WfctHS+XLOL/snNeOCivvRs1yToWCJSDkEWxTygq5l1AjYCVwBXFVgzA7gp/PrFIGCvXp+IbYdy8njswxWkfbKaBnVqMf7ivvwktaNmIkTiWGBF4e65ZnYT8C6hw2OnuvsSMxsbvn8yMJPQobErCR0eOzqovFKyD5du4+4Z6WzYdYhL+idxx4getGxUN+hYIlJBge4sdveZhMog8rbJEZcduDHauaRsNu89xH1vZjArfQtdWjXkpV+czCldjgk6lohUEr2qKOWWm5fPs3PX8bf3lpGb7/z+3O784vTOmokQSTAqCimXr9fvZty0dDI272Nw91bcd0Efko/RTIRIIlJRSJnsPXiUh95dyotfradN43o8eXV/hvVpq5kIkQSmopBS+W4mYvysTHZl5/Bfp3biN0O70UgzESIJT89yKdGyLfu5a3o6X63ZxQkdm/HM6IH06dA06FgiEiUqCilS9pFcJs5ewdRP19CoXi0mXNyXyzUTIVLtqCjkB9ydd9K3cN9bGWzee5grTurIrcN66H0iRKopFYV8z9od2dw9YwmfLN9Or3ZNeOyq/gw4tnnQsUQkQCoKAeDw0Tye/GgVT368iro1a3DPyF5ce/Kx1KqpmQiR6k5FIXy4dBv3zFjC+l0HGXVCe8aN6EnrJvWCjiUiMUJFUY1t3HOI+95cwrtLttKlVUNe/PkgfnRcy6BjiUiMUVFUQ3n5ztOfreGv7y3HcW4d1p2fn6ZTb4hI4VQU1czaHdn87l/fMH/dbs7uGXo70qTmOvWGiBRNRVFN5Oc7z3+xjgmzllKrpvG3y/tx0YkddOoNESmRiqIa2LrvML95ZRGfr9rJGd1a8eAlx9O2qV6sFpHSUVEkuI+Xb+c3ryziUE4e4y/uyxUnddRWhIiUiYoiQeXm5fPX95fz5Eer6N6mMY9ffSLHtW4cdCwRiUMqigS0ee8hfvXi18xft5srB3bknpG9qVe7ZtCxRCROBVIUZtYCeAVIAdYCl7v77gJrOgLPAW2BfCDN3SdGN2n8mbtqJze9uJDDR/OYeMUJjDqhQ9CRRCTOBXXg/O3AbHfvCswOXy8oF/j/7t4TOBm40cx6RTFjXHF3psxZzTVPfUmzBrWZftNpKgkRqRRB7XoaBQwOX34W+Ai4LXKBu28GNocv7zezTKADkBG1lHEi+0gut732LW99u5lhvdvy8OX99IZCIlJpgvpp0iZcBLj7ZjNrXdxiM0sBTgS+LGbNGGAMQHJycuUljXFrdmQz9vkFrNi2n9uG9WDsGZ11VJOIVKoqKwoz+4DQ6wsFjSvj52kEvAbc4u77ilrn7mlAGkBqaqqX5THi1ezMrdzyyiJq1TCe/a+BnN61VdCRRCQBVVlRuPvZRd1nZlvNrF14a6IdsK2IdbUJlcQL7v56FUWNO/n5zqOzVzBp9gr6dGjC5GsG6DQcIlJlgtr1NAO4DpgQ/nN6wQUW2n/yFJDp7n+LbrzYtffgUW555Ws+XLadSwckcf+FfXToq4hUqaCOepoADDWzFcDQ8HXMrL2ZzQyvORW4FjjLzBaFP0YEEzc2ZGzax8jHPuXTlTv404V9+Mulx6skRKTKBbJF4e47gSGF3L4JGBG+/CmgV2XDpn2dxR2vL6ZZ/Tq8csMp9E/W25OKSHToGMoYl5ObzwNvZ/Ds3HUM6tSCx67qT6vGdYOOJSLViIoihm3dd5j/98JCFqzbzc9P68Ttw3voPaxFJOpUFDHqqzW7uPHFhWQfyeXvV57IyH7tg44kItWUiiLGuDtPf7aWP8/MpGOLBrzw80F0a6OzvopIcFQUMeRgTi53vL6Y6Ys2MbRXG/56eT+a1KsddCwRqeZUFDFi7Y5sxv5zAcu27uf353bnl2d0oUYNHfQlIsFTUcSA707FUbOG8czogZzRTafiEJHYoaIIUOSpOHq3D52Ko2MLnYpDRGKLiiIgew7mcMsri/hIp+IQkRinogjAkk17GfvPBWzZe5j7L+zD1YOSdWpwEYlZKoooe21BFn+YtpjmDXQqDhGJDyqKKMnJzedPb2Xw/BfrOLlzC/5+pU7FISLxQUURBVv2HuaXLyzg6/V7GPPjztx6bnedikNE4oaKoorNXbWTX720kEM5eTxxdX9G9G0XdCQRkTJRUVQRd2fKnDVMeGcpxx7TgJfHnMxxrXUqDhGJPyqKKpB9JJdbX/uWt7/dzLDebfnLZcfTWKfiEJE4paKoZKu2H+CG5xewevsBbh/egxt+3FmHvopIXFNRVKJ30rfwu399Q51aNXj++kGcelzLoCOJiFRYIEVhZi2AV4AUYC1wubvvLmJtTWA+sNHdz49WxrLIzcvnr+8v58mPVtEvqSlPXDOADs3qBx1LRKRSBHWM5u3AbHfvCswOXy/KzUBmVFKVw84DR7ju6a948qNVXDUomVfHnqKSEJGEElRRjAKeDV9+FriwsEVmlgScB0yJTqyyWbRhDyP//inz1u7moUuP588X9aVuLZ2vSUQSS1CvUbRx980A7r7ZzFoXse5R4FagxONKzWwMMAYgOTm5kmIW7ZV567nrjSW0alyX18b+iL5JTav8MUVEglBlRWFmHwBtC7lrXCn//vnANndfYGaDS1rv7mlAGkBqaqqXPmnZ5Obl88DMTJ7+bC2nd23JpCtOpHnDOlX1cCIigauyonD3s4u6z8y2mlm78NZEO2BbIctOBS4wsxFAPaCJmf3T3a+posgl2nf4KDe9+DWfLN/O6FNTGDeip07FISIJL6ifcjOA68KXrwOmF1zg7ne4e5K7pwBXAP8JsiTW7czm4ic+5/OVOxh/cV/uGdlbJSEi1UJQP+kmAEPNbAUwNHwdM2tvZjMDylSkuat2Murxz9hx4AjPXz+IKwdW/WsgIiKxIpAXs919JzCkkNs3ASMKuf0j4KMqD1aIV+atZ9y0dFJaNmTKT1NJadkwiBgiIoHRZHYR3J1Js1fyyAfL+XG3Vjx21Yk00fmaRKQaUlEUIi/fuXt6Oi98uZ5L+icx4ZK+1NbrESJSTemn33deeAFSUvAaNdjTuj37pz7LLwd34eHLjldJiEi1pi0KCJXEmDFw8CAGHLNzC397/wlqXXEiWI+g04mIBEq/KgOMGwcHD37vplpHDoVuFxGp5lQUAOvXl+12EZFqREUBUNS5oaJwzigRkVinogB44AFo0OD7tzVoELpdRKSaU1EAXH01pKXBsceCWejPtLTQ7SIi1ZyOevrO1VerGERECqEtChERKZaKQkREiqWiEBGRYqkoRESkWCoKEREplrlX2dtLB8bMtgPrAnjolsCOAB63rOIhpzJWjnjICPGRM9EzHuvurQq7IyGLIihmNt/dU4POUZJ4yKmMlSMeMkJ85KzOGbXrSUREiqWiEBGRYqkoKlda0AFKKR5yKmPliIeMEB85q21GvUYhIiLF0haFiIgUS0UhIiLFUlFUgJm1MLP3zWxF+M/mxaytaWZfm9lbsZbRzDqa2YdmlmlmS8zs5ihlG2Zmy8xspZndXsj9ZmaTwvd/a2b9o5GrHDmvDuf71sw+N7N+sZYxYt1JZpZnZpdGM1/4sUvMaGaDzWxR+Pvw42hnDGco6evd1MzeNLNvwjlHRznfVDPbZmbpRdxf+c8bd9dHOT+Ah4Dbw5dvBx4sZu1vgReBt2ItI9AO6B++3BhYDvSq4lw1gVVAZ6AO8E3BxwRGALMAA04Gvgzga1yanD8CmocvD492ztJkjFj3H2AmcGmsZQSaARlAcvh66xj9ev/hu+cR0ArYBdSJYsYfA/2B9CLur/TnjbYoKmYU8Gz48rPAhYUtMrMk4DxgSnRifU+JGd19s7svDF/eD2QCHao410Bgpbuvdvcc4OVw1kijgOc85AugmZm1q+JcZc7p7p+7++7w1S+ApFjLGPYr4DVgWzTDhZUm41XA6+6+HsDdYzWnA43NzIBGhIoiN1oB3f2T8GMWpdKfNyqKimnj7psh9MMWaF3EukeBW4H8KOWKVNqMAJhZCnAi8GUV5+oAbIi4nsUPy6k0a6paWTNcT+i3uWgqMaOZdQAuAiZHMVek0vw/dgOam9lHZrbAzH4atXT/pzQ5HwN6ApuAxcDN7h7Ec7solf680TvclcDMPgDaFnLXuFL+/fOBbe6+wMwGV2K0yMeoUMaIz9OI0G+ct7j7vsrIVtzDFXJbwWO1S7OmqpU6g5mdSagoTqvSRIU8dCG3Fcz4KHCbu+eFfhGOutJkrAUMAIYA9YG5ZvaFuy+v6nARSpPzXGARcBbQBXjfzOZE4TlTWpX+vFFRlMDdzy7qPjPbambt3H1zeNOusE3lU4ELzGwEUA9oYmb/dPdrYigjZlabUEm84O6vV1a2YmQBHSOuJxH6Da2sa6paqTKY2fGEdi0Od/edUcr2ndJkTAVeDpdES2CEmeW6+xtRSVj6r/cOd88Gss3sE6AfodfMoqU0OUcDEzz0gsBKM1sD9AC+ik7EElX+8yaaLxQl2gfwF77/QvFDJawfTPRfzC4xI6HfQJ4DHo1irlrAaqAT//eiYe8Ca87j+y/KfRXA17g0OZOBlcCPop2vtBkLrH+G6L+YXZr/x57A7PDaBkA60CcGcz4J3Bu+3AbYCLSMcs4Uin4xu9KfN1H7hyXiB3BM+Bt7RfjPFuHb2wMzC1kfRFGUmJHQrhIHviW0Sb0IGBGFbCMI/ba4ChgXvm0sMDZ82YDHw/cvBlID+jqXlHMKsDvi/25+rGUssDbqRVHajMDvCR35lE5oF2gsfr3bA++FvyfTgWuinO8lYDNwlNDWw/VV/bzRKTxERKRYOupJRESKpaIQEZFiqShERKRYKgoRESmWikJERIqlohARkWKpKESqWPi03ovMrL2ZNTCzt81safgU1RMi1v3GzNab2WNB5hUpSKfwEKl6h9z9BAAzawA87O4fmlkdYLaZDXf3We7+iJntJnS6DZGYoS0KkQoys1vN7Nfhy4+Y2X/Cl4eY2T8j17r7QXf/MHw5B1hI9E9LLlImKgqRivsEOD18ORVoFD7J4mnAnKL+kpk1A0YSOrWKSMxSUYhU3AJggJk1Bo4AcwkVxukUURRmVovQOXsmufvqaAUVKQ+9RiFSQe5+1MzWEjr99OeETq54JqH3Ksgs4q+lASvc/dFoZBSpCG1RiFSOT4Dfhf+cQ+hsnou8kLNumtn9QFPglmgGFCkvFYVI5ZgDtAPmuvtW4DCF7HYKv3/6OKAXsDB82OzPo5pUpIy060mkErj7bKB2xPVuRazLovC3qhSJWdqiEKl6+74buCtukZn9BrgDiJX3XhYB0BsXiYhI8bRFISIixVJRiIhIsVQUIiJSLBWFiIgU638AD9Qciqo4H3cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "w2 = []\n",
    "w3 = []\n",
    "for w in single_layer.w_history:\n",
    "    w2.append(w[2])\n",
    "    w3.append(w[3])\n",
    "plt.plot(w2, w3)\n",
    "plt.plot(w2[-1], w3[-1], 'ro')\n",
    "plt.xlabel('w[2]')\n",
    "plt.ylabel('w[3]')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 1)\n",
      "[[5]\n",
      " [6]\n",
      " [7]]\n"
     ]
    }
   ],
   "source": [
    "b2 = np.array([4])\n",
    "temp = np.array([[1],[2],[3]])\n",
    "z2 = temp + b2 # (3,1) + (1,)\n",
    "print(z2.shape)\n",
    "print(z2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 1)\n",
      "(1,)\n",
      "(1,)\n",
      "[3.98]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "b2 = 4\n",
    "err = np.array([[1],[2],[3]])\n",
    "print(err.shape)\n",
    "b2_grad = np.sum(err, axis=0)/3\n",
    "print(b2_grad.shape)\n",
    "b2 = b2 - 0.01*b2_grad\n",
    "print(b2.shape)\n",
    "print(b2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 기본 딥러닝 구현(2층 구조)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DualLayer(SingleLayer):\n",
    "    \n",
    "    def __init__(self, units=10, learning_rate=0.1, l1=0, l2=0):\n",
    "        self.units = units         # 은닉층의 뉴런 개수\n",
    "        self.w1 = None             # 은닉층의 가중치\n",
    "        self.b1 = None             # 은닉층의 절편\n",
    "        self.w2 = None             # 출력층의 가중치\n",
    "        self.b2 = None             # 출력층의 절편\n",
    "        self.a1 = None             # 은닉층의 활성화 출력\n",
    "        self.losses = []           # 훈련 손실\n",
    "        self.val_losses = []       # 검증 손실\n",
    "        self.lr = learning_rate    # 학습률\n",
    "        self.l1 = l1               # L1 손실 하이퍼파라미터\n",
    "        self.l2 = l2               # L2 손실 하이퍼파라미터\n",
    "\n",
    "    def forpass(self, x):\n",
    "        z1 = np.dot(x, self.w1) + self.b1        # 첫 번째 층의 선형 식을 계산합니다\n",
    "        self.a1 = self.activation(z1)            # 활성화 함수를 적용합니다\n",
    "        z2 = np.dot(self.a1, self.w2) + self.b2  # 두 번째 층의 선형 식을 계산합니다.\n",
    "        return z2\n",
    "\n",
    "    def backprop(self, x, err):\n",
    "        m = len(x)       # 샘플 개수\n",
    "        # 출력층의 가중치와 절편에 대한 그래디언트를 계산합니다.\n",
    "        w2_grad = np.dot(self.a1.T, err) / m\n",
    "        b2_grad = np.sum(err, axis=0) / m\n",
    "        # 시그모이드 함수까지 그래디언트를 계산합니다.\n",
    "        err_to_hidden = np.dot(err, self.w2.T) * self.a1 * (1 - self.a1)\n",
    "        # 은닉층의 가중치와 절편에 대한 그래디언트를 계산합니다.\n",
    "        w1_grad = np.dot(x.T, err_to_hidden) / m\n",
    "        b1_grad = np.sum(err_to_hidden, axis=0) / m\n",
    "        return w1_grad, b1_grad, w2_grad, b2_grad\n",
    "\n",
    "    def init_weights(self, n_features):\n",
    "        self.w1 = np.ones((n_features, self.units))  # (특성 개수, 은닉층의 크기)\n",
    "        self.b1 = np.zeros(self.units)               # 은닉층의 크기\n",
    "        self.w2 = np.ones((self.units, 1))           # (은닉층의 크기, 1)\n",
    "        self.b2 = 0\n",
    "        \n",
    "    def fit(self, x, y, epochs=100, x_val=None, y_val=None):\n",
    "        y = y.reshape(-1, 1)          # 타깃을 열 벡터로 바꿉니다.\n",
    "        y_val = y_val.reshape(-1, 1)\n",
    "        m = len(x)                    # 샘플 개수를 저장합니다.\n",
    "        self.init_weights(x.shape[1]) # 은닉층과 출력층의 가중치를 초기화합니다.\n",
    "        # epochs만큼 반복합니다.\n",
    "        for i in range(epochs):\n",
    "            a = self.training(x, y, m)\n",
    "            # 안전한 로그 계산을 위해 클리핑합니다.\n",
    "            a = np.clip(a, 1e-10, 1-1e-10)\n",
    "            # 로그 손실과 규제 손실을 더하여 리스트에 추가합니다.\n",
    "            loss = np.sum(-(y*np.log(a) + (1-y)*np.log(1-a)))\n",
    "            self.losses.append((loss + self.reg_loss()) / m)\n",
    "            # 검증 세트에 대한 손실을 계산합니다.\n",
    "            self.update_val_loss(x_val, y_val)\n",
    "            \n",
    "    def training(self, x, y, m):\n",
    "        z = self.forpass(x)       # 정방향 계산을 수행합니다.\n",
    "        a = self.activation(z)    # 활성화 함수를 적용합니다.\n",
    "        err = (a - y)            # 오차를 계산합니다.\n",
    "        # 오차를 역전파하여 그래디언트를 계산합니다.\n",
    "        w1_grad, b1_grad, w2_grad, b2_grad = self.backprop(x, err)\n",
    "        # 그래디언트에서 페널티 항의 미분 값을 뺍니다\n",
    "        w1_grad += (self.l1 * np.sign(self.w1) + self.l2 * self.w1) / m\n",
    "        w2_grad += (self.l1 * np.sign(self.w2) + self.l2 * self.w2) / m\n",
    "        # 은닉층의 가중치와 절편을 업데이트합니다.\n",
    "        self.w1 -= self.lr * w1_grad\n",
    "        self.b1 -= self.lr * b1_grad\n",
    "        # 출력층의 가중치와 절편을 업데이트합니다.\n",
    "        self.w2 -= self.lr * w2_grad\n",
    "        self.b2 -= self.lr * b2_grad\n",
    "        return a\n",
    "    \n",
    "    def reg_loss(self):\n",
    "        # 은닉층과 출력층의 가중치에 규제를 적용합니다.\n",
    "        return self.l1 * (np.sum(np.abs(self.w1)) + np.sum(np.abs(self.w2))) + \\\n",
    "               self.l2 / 2 * (np.sum(self.w1**2) + np.sum(self.w2**2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.967032967032967"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dual_layer = DualLayer(units=20, l2=0.01)\n",
    "dual_layer.fit(x_train_scaled, y_train, \n",
    "               x_val=x_val_scaled, y_val=y_val, epochs=12000)\n",
    "dual_layer.score(x_val_scaled, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEKCAYAAADn+anLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAuVUlEQVR4nO3dd3xV9f3H8dcnEzIgJCSMhL0CCjLCkiUgKENxi+KiVoqjVVut+GvraGtrW9taK4Xi1roHFRVBxYGICsEiM0zBhJWEmQAh6/v741wwYC4kmJubhPfz8cgjueec783ny8g753zP93vMOYeIiEh5QoJdgIiI1FwKCRER8UshISIifikkRETEL4WEiIj4pZAQERG/AhoSZnauma0xs/VmNqWc/ePMbJmZLTWzdDMbWNG2IiISeBaoeRJmFgqsBUYAWcBi4Arn3Koyx8QA+51zzsy6Aa8451Ir0lZERAIvkGcSfYD1zrmNzrlC4CVgXNkDnHP57ruUigZcRduKiEjghQXwvZOBzDKvs4C+xx5kZhcCfwSSgDGVaetrPwmYBBAdHd0rNTX1BxdeFQqKSth9oIg9BwopLnWEmNGgfhhx9SOIqReGBbtAqRquFA7sgv25UHwQQkIhKgGiGkNYZLCrEzmhJUuW5DrnEv3tD2RIlPdz8HvXtpxzM4GZZjYY+B1wdkXb+trPAGYApKWlufT09JMuOBBKSh1ffrOTWUu3Mnv5NvYVFBMTHcGYbs04v3syPVvGYabIqPWcg80LYfHjsHoWlH4L7YZD359A+xEQontEpGYys83H2x/IkMgCWpR5nQJs9Xewc26+mbUzs8aVbVuThYYYZ7ZrzJntGnP/uNP4eE0Oby7dwouLM3nm880kx9VndNemjOnWnDNSGiowaiszaD3A+8jbDl89C+lPwguXQUJ76DsZul8JEdHBrlSkUgI5cB2GN/g8HNiCN/h8pXNuZZlj2gMbfAPXPYG38AIh9ERty1MTzyT8ySso4r2VO5i9fBvz1+VQVOJIaVSfMV2bMaZbM7omKzBqvZIiWPUmfD4Vtn4F9RpCr+ugzyRomBLs6kQAMLMlzrk0v/sDuQqsmY0GHsb7of+kc+4BM5sM4JybbmZ3AdcARcBB4E7n3AJ/bU/0/WpTSJS192AR76/awTvLtvLpulyKSx0t4uszpmtzxnZrxmnNGygwajPnIHMRfPEv71IUBl3GQb+boEXvYFcnp7ighkR1q60hUdaeA4W8t2oH7yzbxmfrvcBolRDFmK7NGHV6M05PVmDUanu+hUUzYMmzcGgvpPSGfjdC53EQGsirv7VXUVERWVlZFBQUBLuUWq1evXqkpKQQHh5+1HaFRC22e38h763aztvLtrFww05KSh3NGtbj7M5NOLtLE/q1jScyLDTYZcrJOJQHS1+EL6fBro3QIAX63ww9r4HImGBXV6N88803xMbGkpCQoF+QTpJzjp07d5KXl0ebNm2O2qeQqCN27S/kw4xs3l+1nflrczlYVEJMZBhDOiYyoksThnZKomFU+InfSGqW0lJYOwc+fxQ2fwb14qDPDdDnJxDj967EU8rq1atJTU1VQPxAzjkyMjLo3LnzUdsVEnVQQVEJCzfk8v6qHby/Kpvc/EOEhhi9WjViSMdEBndI5LTmDQgJ0X+qWiVzMXz2MGS8482x6D4BzrwF4tsGu7KgWr169fd+sMnJKe/PUiFRx5WWOr7O2sP7q3bw8ZocVm3bB0B8dAQD2zdmcMdEBndoTFKDekGuVCosdx0sfAS+fglKi71B7gG3QvMewa4sKBQSVUchcQqGxLFy8g6xYH0O89fm8um6HHLzCwFIbRrL4I6JDGzfmN6t46kfobGMGi9vO3wxzZtvcWgftBnihUW7Yd68jFOEQqLqKCQUEkcpLXWs3r6P+Wtzmb82hyWbd1NYUkpEWAhprRoxsENjBrZvzGnNGxKqS1M1V8E+WPKUFxh526DZGTD4Tug05pSYyR3skNizZw8vvPACN910U6XajR49mhdeeIG4uLhKtbvuuusYO3Ysl1xySaXaVcTJhITuuavDQkKM05o35LTmDbnxrHYcKCxm8abdLFiXw6frcvnznDX8mTXERYUzoF1jBrRvzKAOjWkRHxXs0qWseg28M4i+k2HZK7Dgb/DyVZDUBQb9Ak670FszSgJiz549/Otf//peSJSUlBAa6v/Pffbs2YEurVooJE4hURHe3VBDOnp3zeTkHWLhhlw+XZfLgnW5vLN8GwCtEqK8wGjvLSeiu6ZqiLBI6Hk1nHEFrJwJnz4Er18PH//RC4uul0Jo3f67uv+tlazauq9K37NL8wbce95pfvdPmTKFDRs20L17d8LDw4mJiaFZs2YsXbqUVatWccEFF5CZmUlBQQG33norkyZNAqB169akp6eTn5/PqFGjGDhwIAsXLiQ5OZk333yT+vXrn7C2efPmcccdd1BcXEzv3r2ZNm0akZGRTJkyhVmzZhEWFsbIkSN56KGHePXVV7n//vsJDQ2lYcOGzJ8/v0r+fBQSp7DE2EjGdU9mXPdknHNsyNnPgnU5LFify6ylW3nhy28JMeia3NB3aSqRnq3iNDcj2ELDoNulcPrFkPEWzP8L/PdGLywG3u7dFaUVaKvMgw8+yIoVK1i6dCkff/wxY8aMYcWKFUfmGzz55JPEx8dz8OBBevfuzcUXX0xCQsJR77Fu3TpefPFFHnvsMS677DJef/11rrrqquN+34KCAq677jrmzZtHx44dueaaa5g2bRrXXHMNM2fOJCMjAzNjz549APz2t79l7ty5JCcnH9lWFRQSAoCZ0T4phvZJMVw3oA1FJaV8nbnHO8tYn8v0TzYy9aMN1A8PpU+beAZ18C5PpTaN1f3rwRIS4t351Pl8WPcefPJnePt2+OQv3uWpntdARN26dHi83/irS58+fY6akPbII48wc+ZMADIzM1m3bt33QqJNmzZ0794dgF69erFp06YTfp81a9bQpk0bOnbsCMC1117L1KlTueWWW6hXrx4//vGPGTNmDGPHjgVgwIABXHfddVx22WVcdNFFVdBTj0JCyhUeGkJa63jSWsdz+4iO5BUU8cXGXUfONH7/zmoAGsdEMqB9AgPbN2ZQh0SaNtStttXODDqeAx1GwsaPYf5DMOcu+PSvMOjn0GsihOvvpapER3+3ku/HH3/MBx98wOeff05UVBRnnXVWucuHREZ+d2YXGhrKwYMHT/h9/N1UFBYWxqJFi5g3bx4vvfQSjz76KB9++CHTp0/nyy+/5J133qF79+4sXbr0e2F1MhQSUiGx9cIZ0aUJI7o0AWDrnoMsWJ/LZ76PN5d6K7m3T4phaKdEhqU2Ia11I8JD6/7dNzWGGbQb6n1sXuhdfpozBT57BAb/AnpcA2ERwa6y1omNjSUvL6/cfXv37qVRo0ZERUWRkZHBF198UWXfNzU1lU2bNrF+/Xrat2/Pc889x5AhQ8jPz+fAgQOMHj2afv360b59ewA2bNhA37596du3L2+99RaZmZkKCQme5nH1uSytBZeltaC01JGxPY/P1ucyf10OzyzczGOffkNsPW+gfHjnJIZ0TCI+Wj+gqk2rM+Hat+Cb+fDhA/DOL2DBw96ts92vrPMD3FUpISGBAQMGcPrpp1O/fn2aNGlyZN+5557L9OnT6datG506daJfv35V9n3r1avHU089xaWXXnpk4Hry5Mns2rWLcePGUVBQgHOOv//97wDceeedrFu3Duccw4cP54wzzqiSOjRPQqpc/qFiFqzL5cOMHXyYkUNu/iFCDHq2bMSwzkkMT21CxyYxGsuoLs7Bhg/howdgyxJo1AaG3AXdLqsVt84Ge55EXaLJdAqJGqe01LF8y14+zMjmw4xslm/ZC0ByXH2GpSYxrHMS/dsmUC+85v+wqvWcg7VzvbDYvgwSOsBZU+C0i2r0pDyFRNVRSCgkarwd+wr4KCObeRnZLFjnrWYbERZC95Q4erdpRO/W8fRq1YjYerocEjDOweq3vDGL7FXQpCucfR+0H14jl/uoqyFx880389lnnx217dZbb2XixIkB+54KCYVErVJQVMIXG3fy2fpcFm3azYoteykpdYSYN8HpjJQ4uiY3pGtKQzo2idUgeFUrLYWVb8CHv4Pdm6D1IBhxPyT3CnZlR6mrIREMWpZDapV64aGc1SmJszolAbD/UDH/+3YPizbtIn3TLmZ9vZXnv/wWgIiwEDo3a0C35IZ0ad6Ajk1i6dQ0lphI/RM+aSEh0PUSb57Fkqfhkz/BY8OgywUw/B5IaBfsCqUG0P8wqTGiI8O8md0dGgPeeMa3uw6wbMtelmftYfmWvcz83xae+2LzkTbJcfVJbRpLx6ax3ucmsbRNjNas8MoIi4C+k6D7FbDwUVj4T8h4G3pe6w1wxzY58XtInaWQkBorJMRo3Tia1o2jOf+M5oAXHFv2HGTN9jzW7MjzPm/P45O1ORSXepdOQ0OMVglRtE+MOTKLvH1SDO0SY4jWmYd/kbEw9G7ofb03e3vJU/D1i9D/Fm8Gtx6rekrS/xipVUJCjBbxUbSIj+LsLt/9hltYXMo3uft9wbGP9dn5rM/O58OM7CPhAdC8YT3a+QKjbIAkREfoltzDYpJgzEPQ70b48Pcw/8/w1TMw/F5vccEafCeUVD2FhNQJEWEhdGrqjVPgO+sAKCopZfPO/UdCY0OO9/Ur6ZkcKCw5clxcVPhRZx7tkmJonxhDclz9U/cxsAnt4NKnoN9N3sztN2+CRTPg3D96k/WkXDExMeTn55e7b9OmTYwdO5YVK1ZUc1UnTyEhdVp4aAjtk2JpnxR71PbSUse2fQVHwmN9dj4bsvN5b9UOXlqceeS4euEhtG38XXh0bBJD+6RYWidEEXaq3G3Vojdc/z6seB0+uBeeGuUNbo+4Hxq1DnZ1EmAKCTklhYQYyXH1SY6rf+T5Goft2l/Ihpz8owJkyebdzPp665FjIkJDaJsYTYcmsXTwhUeHJrG0iq+j4RES4i1PnjrGG9j+7GFY8y70v9lbRDAy9oRvUSXenQLbl1ftezbtCqMe9Lv7rrvuolWrVkceOnTfffdhZsyfP5/du3dTVFTE73//e8aNG1epb1tQUMCNN95Ieno6YWFh/O1vf2Po0KGsXLmSiRMnUlhYSGlpKa+//jrNmzfnsssuIysri5KSEn7zm99w+eWX/6BuV5RCQuQY8dERxEfH07t1/FHbDxQWsyF7P2t35LE2O491O/JZmrmbt/yER8ckLzg6NImpO+EREQVn3QU9roJ5v/Wekrf0eRjxO2+Zjzo4rjN+/Hhuu+22IyHxyiuvMGfOHG6//XYaNGhAbm4u/fr14/zzz6/UuNbUqVMBWL58ORkZGYwcOZK1a9cyffp0br31ViZMmEBhYSElJSXMnj2b5s2b88477wDewoLVRSEhUkFREWF0TfEm95V1oLCY9dn5rN2RzzpfePzv22PCIyyEto2j6eg780ht1oDOzWJJjqtfOwfMGybDRf+GPpPg3Tth5iT46llvwDspgBPfjvMbf6D06NGD7Oxstm7dSk5ODo0aNaJZs2bcfvvtzJ8/n5CQELZs2cKOHTto2rRphd93wYIF/PSnPwW8FV9btWrF2rVr6d+/Pw888ABZWVlcdNFFdOjQga5du3LHHXdw1113MXbsWAYNGhSo7n6PQkLkB4qKCKNbShzdUuKO2r7/UDEbcnzhsSOPtTvy+Orboy9bxdYLo3NTLzA6N2tA52YN6NQ0tvasZZXSC67/wLv76YP7YPpAb6B7yF116pbZSy65hNdee43t27czfvx4nn/+eXJycliyZAnh4eG0bt263OdIHI+/1S6uvPJK+vbtyzvvvMM555zD448/zrBhw1iyZAmzZ8/m7rvvZuTIkdxzzz1V0bUTCmhImNm5wD+AUOBx59yDx+yfANzle5kP3Oic+9q3bxOQB5QAxcebNi5SE0VHlh8e+YeKWbM9j9Xb9h35eHVJ1pG7rUIM2jSOPhIaXXyfmzSIrJlnHSEhkDbRm7n9wb2w8BFvkPucP3hPzquJNVfS+PHjueGGG8jNzeWTTz7hlVdeISkpifDwcD766CM2b9584jc5xuDBg3n++ecZNmwYa9eu5dtvv6VTp05s3LiRtm3b8rOf/YyNGzeybNkyUlNTiY+P56qrriImJoann3666jvpR8BCwsxCganACCALWGxms5xzq8oc9g0wxDm328xGATOAvmX2D3XO5QaqRpFgiIkMo1erRvRq1ejItsOzyzO272PVNi9Almbu4e1l244cEx8dwenJDenmW8+qW0pDmjaoV3OCIzoBxj0KPa72nl/x6rXQbjiM+SvEtzlx+xrstNNOIy8vj+TkZJo1a8aECRM477zzSEtLo3v37qSmplb6PW+66SYmT55M165dCQsL4+mnnyYyMpKXX36Z//znP4SHh9O0aVPuueceFi9ezJ133klISAjh4eFMmzYtAL0sX8AW+DOz/sB9zrlzfK/vBnDO/dHP8Y2AFc65ZN/rTUBaZUJCC/xJXbOvoIgMX2is3LqX5Vv2sXZHHiW+CYKNYyLpltKQrsleaHRNaUhSbA14VGlJMSx+3JuM50pg2G+g709O6vkVWuCv6tS0Bf6Sgcwyr7M4+izhWNcD75Z57YD3zMwB/3bOzaj6EkVqtgb1wunTJp4+bb6706qgqIRV2/axPGsvy7L2snzLHj5ek83hieVNG9Sja0rDI2crXZMbVv8YR2gY9JsMncfC27fD3Lu9FWfPfxSSKv9btwRPIEOivHPgck9bzGwoXkgMLLN5gHNuq5klAe+bWYZzbn45bScBkwBatmz5w6sWqeHqhYfSs2Ujerb87nLVgcJiVm3d5wuNvSzN3MP7q3YAEB5qnNb8u9Do1aoRTRpU09lGwxS48hVY/iq8exf8e5D3CNWBt9fpR6guX76cq6+++qhtkZGRfPnll0Gq6OQF/XKTmXUDZgKjnHNr/bzXfUC+c+6h431PXW4S+c7O/EN89e0elmzezVebd/N11h4OFZcC3uq5vVo1om/bePq3TaBN4+jAj23k58C7v/TOKJp0hYsfq9DtsqtXryY1NbXmjL3UUs45MjIyas5Dh8wsDFgLDAe2AIuBK51zK8sc0xL4ELjGObewzPZoIMQ5l+f7+n3gt865Ocf7ngoJEf8Ki0tZtW0fSzbvZsnmXaRv2k123iHAu0TVr208Z7ZrTP92CbSIjwpcIRnvwKyfQWE+jPitN9fiOAHwzTffEBsbS0JCgoLiJDnn2LlzJ3l5ebRpc/RNBEF9Mp2ZjQYexrsF9knn3ANmNtlX9HQzexy4GDh8/1ixcy7NzNrinV2Ad0nsBefcAyf6fgoJkYpzzrExdz+fb9jJ5xt38sWGnezcXwh4Zxr92yUwuGMigzs0Ji4qomq/eX42vHkzrHsP2p8N4/7l97kVRUVFZGVlVXoeghytXr16pKSkEB5+9GU+Pb5URCrEOce67HwvNHzBsfdgESEGPVo2YminRM7qlESXZg2qZmVc57w7oN77NUREw4UzoMPZP/x9pVIUEiJyUkpKHUsz9/DJmmw+WpPD8i3eekGJsZEM6ZjIyC5NGNwx8YffOZWzBl77EexYCWfd7Q1s65kV1UYhISJVIifvEJ+szeHjNdnMX5vDvoJioiJCGZqaxKjTmzK0U9LJP/mv8IB3q+yyl6DDSLhoBtRvdOJ28oMpJESkyhWVlPL5hp28u2I776/aTm5+IRFhIQzukMj53ZszskuTyp9hOAfpT3jLgTdMhgmvQeMOgemAHKGQEJGAKil1pG/axbsrtjNnxXa27ysgNjKMUV2bclHPFPq0jq/cGEbmInjxCm+m9vgXoVX/wBUvCgkRqT6lpY4vvtnJG19t4d3l29hfWEJyXH0u6pnM+D4tSY6rX7E32vUNPH8J7MmEC6fD6RcFtvBTmEJCRILiYGEJ763azutfbeHTdTkYMLxzE67u14qB7Ruf+OziwC546Ur49gvfwoFXVUvdpxqFhIgEXeauA7y46FteXpzJzv2FtE6I4pr+rbm8d4vjD3YXHfSCYsOHcN4/oNd11VbzqUIhISI1xqHiEuas2M5zn28mffNu4qLCue7M1lzbvzWNov1M2CsqgJcnwPoP4LxHoNe11Vt0HaeQEJEaacnm3Uz7eAMfrN5B/fBQJvRtyY1ntSMhJvL7BxcfgpcmwIZ5cNlz3uqyUiUUEiJSo63dkcf0jzfw36VbiIoI44ZBbfnxoDbfvwxVuB+eOc+bdHf1f3XXUxVRSIhIrbA+O5+H5q5hzsrtNI6J4PYRHRnfuyWhZQe49++EJ0fC/lyY9BHEtw1ewXXEiUJCc99FpEZonxTD9Kt7MfOmM2mbGMOvZq7gomkLWeFbDgTwHpE64TXv65eu8s4uJKAUEiJSo/Ro2YiXJ/Xj4cu7s2X3Qc5/dAH3zVrJwcIS74D4NnDJE5C9yltyvA5dDamJFBIiUuOYGRf0SGbeL4YwoW8rnl64ibH//PS7s4r2Z8Pw38CK17ylPCRgFBIiUmM1rB/O7y44ned/3Jf9h0q4YOpnTP9kA845GPhzLyzm/hpyyn2opVQBhYSI1HgD2jdmzm2DOOe0pjz4bgY3v/AV+wtLYNxUiIiCN34MxYXBLrNOUkiISK0QFxXBo1f24FejOzNnxXYunraQzKIG3gS7bV/DJw8Gu8Q6SSEhIrWGmXHD4LY886M+bN1zkEunf87a+CHQfQJ89g/YviLYJdY5CgkRqXUGdUjklcn9KXWOS6d/zrIud0C9OJj1UygtCXZ5dYpCQkRqpdSmDXj9xjOJiwpnwvPr+LbvvbD1K1g0I9il1SkKCRGptVrER/HSpH40qB/OuI+bsL/lUJj3O9jzbbBLqzMUEiJSqzVrWJ8XbuhLeFgoV267jFJXCrPv1CS7KqKQEJFar1VCNM9e34cNhQk8FXEFrJ0Dq98Kdll1gkJCROqE1KYN+OcVPXhw91AyI9rh3v0lFOwLdlm1nkJCROqMoalJ3DX6dG7JuxbytsOHvw92SbWeQkJE6pTrB7ahzRmDea7kbNyiGbBlSbBLqtUUEiJSp5gZD1zYlVcbTiSXOIrevBVKioNdVq2lkBCROic6Moy/XDWI35VcS3j2ckq/nB7skmqtgIaEmZ1rZmvMbL2ZTSln/wQzW+b7WGhmZ1S0rYjI8aQ2bcCA837EvJIeFH/we9iTGeySaqWAhYSZhQJTgVFAF+AKM+tyzGHfAEOcc92A3wEzKtFWROS4Luvdks86/JLiklJ2vXar5k6chECeSfQB1jvnNjrnCoGXgHFlD3DOLXTO7fa9/AJIqWhbEZETMTN+cflInq13BfFZ89j3vzeCXVKtE8iQSAbKnt9l+bb5cz3wbmXbmtkkM0s3s/ScnJwfUK6I1EXRkWEMufoeVrnW8PbtlO7ZEuySapVAhoSVs63ccz0zG4oXEndVtq1zboZzLs05l5aYmHhShYpI3dY5JYGNg/9BWEkB2566WivFVkIgQyILaFHmdQqw9diDzKwb8Dgwzjm3szJtRUQqasywIbze7Ock713Clv/eG+xyao1AhsRioIOZtTGzCGA8MKvsAWbWEngDuNo5t7YybUVEKsPMuHDiHbwbNozkZf8kL/2VYJdUKwQsJJxzxcAtwFxgNfCKc26lmU02s8m+w+4BEoB/mdlSM0s/XttA1Soip4aYyDBaXTOdJa4TkW/fROGmL4JdUo1nrg7dEpaWlubS09ODXYaI1HBzF60g9e0LiQ8vJOYnc7CkzsEuKWjMbIlzLs3ffs24FpFTzjl9TufTvtM5WAz7HxsDueuCXVKNpZAQkVPShNHDeL7TIxQUFpE/YxRkZwS7pBpJISEipyQz42fjz+fJtg9z8FAhBTNGwObPg11WjaOQEJFTVmiI8YurL2Jq2+lsKYym+OnzKV2uWdllKSRE5JQWGmL8+qpzebnbYywtaU3I6xMpnvN/Wl7cRyEhIqe8sNAQ7r54IEvOeo5nikcQ9sVUCp4YA/u2Bbu0oFNIiIjgjVH8ZFgqTa94lLu5Bbf1Kwof7QcrTu3LTwoJEZEyzjmtKZN/+n/8rOE/WVnQGF6bSNErE+HArmCXFhQKCRGRY7RKiOaft1zK+/2f4a/Fl2Kr3qTwH2mw7JVT7pkUCgkRkXLUCw/ll6NOZ/hPHuLm6L+x8mAjeOMGDj55PuzcEOzyqo1CQkTkOLq3iOOR26/my6Evcn/p9RR9m07xo/0ofP/3UHgg2OUFnEJCROQEIsNCmTy0I5Pv+AMPdfwP7xSnEfHZX9j/1+4ULX25Tl+CUkiIiFRQkwb1+O2E4bS84QXuTXiIDQejCP/vJHY+MoSSzLq5uKhCQkSkknq0bMR9t/yYPRPm8nD0bZTs2kzoE8PJevKaOvd4VC0VLiLyAzjneO9/69k5549cfOhNnIWyqfMkOlxwN6GR0cEu74S0VLiISACZGef07MDlU57g03NmsyisF6mr/8nOB7uR/vYMSkpKg13iD6KQEBGpAqEhxtln9mXg/81m4eBn2WsNSEu/k4wH+vHJvNmUlNbOqzYKCRGRKhQSYpw5bBzt/m8xy3o9QDOXzZBPr+DDP4zj3c8WU1zLziw0JiEiEkClB/fxzZsP0CLjCUqc8XLEhTQacQdj0zoQGmLBLu+EYxIKCRGRalC6axM7Zt5Ns8zZbHPxPFP/WrqOuoFRXZsTEsSw0MC1iEgNEBLfmmbXv0jpdXOIim/OlIK/k/z6WO74+2N8sGoHNfUXdoWEiEg1Cmndn4Y//ZTScdPoFJXP3/Lu5MCL13LDP2fy6bqcGhcWCgkRkeoWEkJIjyup//OllA76JaPDv2Lqrp+w9Jk7uGb6RyzN3BPsCo/QmISISLDtyaTk/fsIXfka20ng3sKriTx9HL8clUpKo6iAfusqGZMws1vNrIF5njCzr8xsZNWVKSJyCotrQeilT8CP3iMxqSn/jniYizJ+zrV/fYU/vruafQVFQSutopebfuSc2weMBBKBicCDAatKRORU1LIvoT+ZD+f8gcGRa5kTfifhC/7KiD+/z7OfbwrKHIuKhsTh+7NGA085574us01ERKpKaBj0v5mQWxYT3nkUd4S/yhshdzLnrZc579HPSN9UvY9RrWhILDGz9/BCYq6ZxQInjDQzO9fM1pjZejObUs7+VDP73MwOmdkdx+zbZGbLzWypmWmgQUROLQ2T4bJnYcLrNI8N44WIP3Db3j8xafpc7nj1a3LzD1VLGRUauDazEKA7sNE5t8fM4oEU59yy47QJBdYCI4AsYDFwhXNuVZljkoBWwAXAbufcQ2X2bQLSnHO5Fe2MBq5FpE4qOgif/g234O8cDIninoIJvBc2hDvOSWVC31Y/aOZ2VU2m6w+s8QXEVcCvgb0naNMHWO+c2+icKwReAsaVPcA5l+2cWwwEb1RGRKSmC68Pw36FTf6UqKadeCjsX/wn8s/MmPUR5z+6gK8DeMtsRUNiGnDAzM4AfglsBp49QZtkILPM6yzftopywHtmtsTMJvk7yMwmmVm6maXn5ORU4u1FRGqZpM7wo7kw+iG6ujV8HHU3Z+99jYlPfs6BwuKAfMuKhkSx865LjQP+4Zz7BxB7gjblnf9UZlLGAOdcT2AUcLOZDS7vIOfcDOdcmnMuLTExsRJvLyJSC4WEQJ8bsJu/JKztYG4veZr5CX8gisCMUYRV8Lg8M7sbuBoY5BtvCD9BmyygRZnXKcDWihbmnNvq+5xtZjPxLl/Nr2h7EZE6rWEKXPkyrHyDmMzFEBGYp+BV9EzicuAQ3nyJ7XiXjf5ygjaLgQ5m1sbMIoDxwKyKfDMzi/bdQYWZRePNz1hRwVpFRE4NZnD6xTAqcNPWKnQm4ZzbbmbPA73NbCywyDl33DEJ51yxmd0CzAVCgSedcyvNbLJv/3QzawqkAw2AUjO7DegCNAZmmtnhGl9wzs05qR6KiMhJq+gtsJfhnTl8jDfWMAi40zn3WkCrqyTdAisiUjknugW2omMSvwJ6O+eyfW+aCHwA1KiQEBGRqlXRMYmQwwHhs7MSbUVEpJaq6JnEHDObC7zoe305MDswJYmISE1R0YHrO83sYmAA3pjEDOfczIBWJiIiQVfRMwmcc68DrwewFhERqWGOGxJmlkf5s6QNcM65BgGpSkREaoTjhoRz7kRLb4iISB2mO5RERMQvhYSIiPilkBAREb8UEiIi4pdCQkRE/FJIiIiIXwoJERHxSyEhIiJ+KSRERMQvhYSIiPilkBAREb8UEiIi4pdCQkRE/FJIiIiIXwoJERHxSyEhIiJ+KSRERMQvhYSIiPilkBAREb8UEiIi4ldAQ8LMzjWzNWa23symlLM/1cw+N7NDZnZHZdqKiEjgBSwkzCwUmAqMAroAV5hZl2MO2wX8DHjoJNqKiEiABfJMog+w3jm30TlXCLwEjCt7gHMu2zm3GCiqbFsREQm8QIZEMpBZ5nWWb1uVtjWzSWaWbmbpOTk5J1WoiIiUL5AhYeVsc1Xd1jk3wzmX5pxLS0xMrHBxIiJyYoEMiSygRZnXKcDWamgrIiJVJJAhsRjoYGZtzCwCGA/Mqoa2IiJSRcIC9cbOuWIzuwWYC4QCTzrnVprZZN/+6WbWFEgHGgClZnYb0MU5t6+8toGqVUREymfOVXSYoOZLS0tz6enpwS5DRKTWMLMlzrk0f/s141pERPxSSIiIiF8KCRER8UshISIifikkRETEL4WEiIj4pZAQERG/FBIiIuKXQkJERPxSSIiIiF8KCRER8UshISIifikkRETEL4WEiIj4pZAQERG/FBIiIuKXQkJERPxSSIiIiF8KCRER8UshISIifikkRETEL4WEiIj4pZAQERG/FBIiIuKXQkJERPxSSIiIiF8KCRER8SugIWFm55rZGjNbb2ZTytlvZvaIb/8yM+tZZt8mM1tuZkvNLD2QdYqISPnCAvXGZhYKTAVGAFnAYjOb5ZxbVeawUUAH30dfYJrv82FDnXO5gapRRESOL5BnEn2A9c65jc65QuAlYNwxx4wDnnWeL4A4M2sWwJpERKQSAhkSyUBmmddZvm0VPcYB75nZEjObFLAqRUTEr4BdbgKsnG2uEscMcM5tNbMk4H0zy3DOzf/eN/ECZBJAy5Ytf0i9IiJyjECeSWQBLcq8TgG2VvQY59zhz9nATLzLV9/jnJvhnEtzzqUlJiZWUekiIgKBDYnFQAcza2NmEcB4YNYxx8wCrvHd5dQP2Ouc22Zm0WYWC2Bm0cBIYEUAaxURkXIE7HKTc67YzG4B5gKhwJPOuZVmNtm3fzowGxgNrAcOABN9zZsAM83scI0vOOfmBKpWEREpnzl37DBB7ZWWlubS0zWlQkSkosxsiXMuzd9+zbgWERG/FBIiIuKXQkJERPxSSIiIiF8KCRER8UshISIifikkRETEL4WEiIj4pZAQERG/FBIiIuKXQkJERPxSSIiIiF8KCRER8UshISIifikkRETEL4WEiIj4pZAQERG/FBIiIuKXQkJERPxSSIiIiF8KCRER8UshISIifikkRETEL4WEiIj4pZAQERG/FBIiIuKXQkJERPxSSIiIiF8BDQkzO9fM1pjZejObUs5+M7NHfPuXmVnPirYVEZHAC1hImFkoMBUYBXQBrjCzLsccNgro4PuYBEyrRFsREQmwQJ5J9AHWO+c2OucKgZeAccccMw541nm+AOLMrFkF24qISICFBfC9k4HMMq+zgL4VOCa5gm0BMLNJeGchAPlmtuYk620M5J5k25qmrvSlrvQD1JeaqK70A35YX1odb2cgQ8LK2eYqeExF2nobnZsBzKhcad9nZunOubQf+j41QV3pS13pB6gvNVFd6QcEti+BDIksoEWZ1ynA1goeE1GBtiIiEmCBHJNYDHQwszZmFgGMB2Ydc8ws4BrfXU79gL3OuW0VbCsiIgEWsDMJ51yxmd0CzAVCgSedcyvNbLJv/3RgNjAaWA8cACYer22gavX5wZesapC60pe60g9QX2qiutIPCGBfzLlyL/WLiIhoxrWIiPinkBAREb9O+ZCoDct/mFkLM/vIzFab2Uozu9W3Pd7M3jezdb7Pjcq0udvXpzVmdk6Z7b3MbLlv3yNmVt7txoHuT6iZ/c/M3q7l/Ygzs9fMLMP3d9O/Fvfldt+/rRVm9qKZ1astfTGzJ80s28xWlNlWZbWbWaSZvezb/qWZta7GfvzF9+9rmZnNNLO4au+Hc+6U/cAbFN8AtMW77fZroEuw6yqnzmZAT9/XscBavOVK/gxM8W2fAvzJ93UXX18igTa+Pob69i0C+uPNRXkXGBWE/vwceAF42/e6tvbjGeDHvq8jgLja2Be8yavfAPV9r18BrqstfQEGAz2BFWW2VVntwE3AdN/X44GXq7EfI4Ew39d/CkY/qvU/VU378P1Bzi3z+m7g7mDXVYG63wRGAGuAZr5tzYA15fUD7y6x/r5jMspsvwL4dzXXngLMA4bxXUjUxn40wPvBasdsr419ObzCQTzeHY9v+3441Zq+AK2P+eFaZbUfPsb3dRjezGarjn4cs+9C4Pnq7sepfrnJ37IgNZbvFLEH8CXQxHnzSvB9TvIddrzlTrLK2V6dHgZ+CZSW2VYb+9EWyAGe8l06e9zMoqmFfXHObQEeAr4FtuHNV3qPWtiXMqqy9iNtnHPFwF4gIWCV+/cjvDODo2ryCVg/TvWQqPDyHzWBmcUArwO3Oef2He/QcrZVarmTQDCzsUC2c25JRZuUsy3o/fAJw7s0MM051wPYj3dZw58a2xff9fpxeJctmgPRZnbV8ZqUs61G9KUCTqb2oPfLzH4FFAPPH95UzmEB6cepHhIVWTqkRjCzcLyAeN4594Zv8w7zVs3F9znbt91fv7J8Xx+7vboMAM43s014K/sOM7P/UPv6ga+GLOfcl77Xr+GFRm3sy9nAN865HOdcEfAGcCa1sy+HVWXtR9qYWRjQENgVsMqPYWbXAmOBCc53rYhq7MepHhK1YvkP390JTwCrnXN/K7NrFnCt7+tr8cYqDm8f77uboQ3e8zoW+U6788ysn+89rynTJuCcc3c751Kcc63x/qw/dM5dVdv64evLdiDTzDr5Ng0HVlEL+4J3mamfmUX5ahgOrKZ29uWwqqy97HtdgvfvtlrOJMzsXOAu4Hzn3IEyu6qvH9UxqFSTP/CWBVmLd3fAr4Jdj58aB+KdFi4Dlvo+RuNdT5wHrPN9ji/T5le+Pq2hzB0mQBqwwrfvUQI0AFeBPp3FdwPXtbIfQHcg3ff38l+gUS3uy/1Ahq+O5/DumqkVfQFexBtLKcL7bfn6qqwdqAe8ird80CKgbTX2Yz3eOMLh//fTq7sfWpZDRET8OtUvN4mIyHEoJERExC+FhIiI+KWQEBERvxQSIiLil0JCpAYws7PMtyquSE2ikBAREb8UEiKVYGZXmdkiM1tqZv8279kY+Wb2VzP7yszmmVmi79juZvZFmWcBNPJtb29mH5jZ17427XxvH2PfPZ/i+cPPARAJJoWESAWZWWfgcmCAc647UAJMAKKBr5xzPYFPgHt9TZ4F7nLOdQOWl9n+PDDVOXcG3hpJ23zbewC34T0roC3eWlciQRUW7AJEapHhQC9gse+X/Pp4C8eVAi/7jvkP8IaZNQTinHOf+LY/A7xqZrFAsnNuJoBzrgDA936LnHNZvtdL8Z4tsCDgvRI5DoWESMUZ8Ixz7u6jNpr95pjjjrfWzfEuIR0q83UJ+v8pNYAuN4lU3DzgEjNLgiPPUW6F9//oEt8xVwILnHN7gd1mNsi3/WrgE+c9ByTLzC7wvUekmUVVZydEKkO/qYhUkHNulZn9GnjPzELwVuu8Ge+BQ6eZ2RK8p31d7mtyLTDdFwIbgYm+7VcD/zaz3/re49Jq7IZIpWgVWJEfyMzynXMxwa5DJBB0uUlERPzSmYSIiPilMwkREfFLISEiIn4pJERExC+FhIiI+KWQEBERv/4fOENKNrwWFB8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.ylim(0, 0.3)\n",
    "plt.plot(dual_layer.losses)\n",
    "plt.plot(dual_layer.val_losses)\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train_loss', 'val_loss'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.0004963243164915432\n",
      "0.9995632217211148\n"
     ]
    }
   ],
   "source": [
    "a = np.random.normal(0, 1,  (2000,3000))\n",
    "# print(a)\n",
    "print(np.mean(a))\n",
    "print(np.std(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RandomInitNetwork(DualLayer):\n",
    "    \n",
    "    def init_weights(self, n_features):\n",
    "        np.random.seed(42)\n",
    "        self.w1 = np.random.normal(0, 1, \n",
    "                                   (n_features, self.units))  # (특성 개수, 은닉층의 크기)\n",
    "        self.b1 = np.zeros(self.units)                        # 은닉층의 크기\n",
    "        self.w2 = np.random.normal(0, 1, \n",
    "                                   (self.units, 1))           # (은닉층의 크기, 1)\n",
    "        self.b2 = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_init_net = RandomInitNetwork(l2=0.01)\n",
    "random_init_net.fit(x_train_scaled, y_train,\n",
    "                    x_val=x_val_scaled, y_val=y_val, epochs=2000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAs6ElEQVR4nO3deXxddZ3/8dfn7tnTtOleukChFAqFlgKCOMqArBYFoazCjPJjEAf8CT/wwcio4zwe+tNxxhmBDjKICwOoiHbGahVG4IeItsUCLXQJtUu6pmmz5+Yu+f7+OCfpTZqkSZt7b9L7fj4eeZz93M89ae8733Pu+R5zziEiIoUrkO8CREQkvxQEIiIFTkEgIlLgFAQiIgVOQSAiUuBC+S5gqMaNG+dmzJiR7zJEREaV1atX73POVfe1bNQFwYwZM1i1alW+yxARGVXMbGt/y3RqSESkwCkIREQKnIJARKTAjbprBCJy7Ekmk9TW1hKPx/NdyqgXi8WYOnUq4XB40NsoCEQk72praykrK2PGjBmYWb7LGbWcc9TX11NbW8vMmTMHvZ1ODYlI3sXjccaOHasQOEpmxtixY4fcslIQiMiIoBAYHkdyHAsmCDbsbuaffr2B+paOfJciIjKiFEwQbK5r4d/+p4Y6BYGISA8FEwTRsPdW48nOPFciIiNNQ0MDjzzyyJC3u+yyy2hoaBjydrfeeis/+clPhrxdthRMEMRCQQA6kuk8VyIiI01/QZBOD/x5sXz5ciorK7NUVe4UzNdHo2EvCOIptQhERrIv/dc63tnZNKz7nDu5nL+/8pR+lz/wwAO89957zJ8/n3A4TGlpKZMmTWLNmjW88847XHXVVWzfvp14PM7dd9/N7bffDhzs+6ylpYVLL72U888/n9dee40pU6bw85//nKKiosPW9uKLL3LvvfeSSqU466yzePTRR4lGozzwwAMsW7aMUCjExRdfzDe+8Q1+/OMf86UvfYlgMEhFRQWvvPLKsByfggmCWPepIbUIRKSnr371q6xdu5Y1a9bw0ksvcfnll7N27dru7+I/8cQTVFVV0d7ezllnncXVV1/N2LFje+xj06ZNPP3003znO9/h2muv5bnnnuOmm24a8HXj8Ti33norL774IieeeCK33HILjz76KLfccgvPP/8869evx8y6Tz99+ctfZsWKFUyZMuWITkn1p2CCIOqfGlIQiIxsA/3lniuLFi3qcUPWv/7rv/L8888DsH37djZt2nRIEMycOZP58+cDsGDBArZs2XLY19mwYQMzZ87kxBNPBOATn/gEDz/8MHfddRexWIxPfvKTXH755VxxxRUAnHfeedx6661ce+21fOxjHxuGd+opnGsEfougQxeLReQwSkpKusdfeuklXnjhBX7/+9/z5ptvcsYZZ/R5w1Y0Gu0eDwaDpFKpw76Oc67P+aFQiD/+8Y9cffXV/OxnP+OSSy4BYOnSpXzlK19h+/btzJ8/n/r6+qG+tb5fb1j2MgrE/GsEHSm1CESkp7KyMpqbm/tc1tjYyJgxYyguLmb9+vW8/vrrw/a6c+bMYcuWLdTU1HDCCSfwgx/8gA984AO0tLTQ1tbGZZddxjnnnMMJJ5wAwHvvvcfZZ5/N2WefzX/913+xffv2Q1omRyKrQWBmlwDfAoLA4865r/Zafh9wY0YtJwPVzrn9w11LVxDo66Mi0tvYsWM577zzOPXUUykqKmLChAndyy655BKWLl3KaaedxkknncQ555wzbK8bi8X47ne/y8c//vHui8V33HEH+/fvZ/HixcTjcZxz/PM//zMA9913H5s2bcI5x4UXXsjpp58+LHVYf02To96xWRDYCFwE1AIrgeudc+/0s/6VwGedcx8aaL8LFy50R/KEsmS6k9kP/pLPXXQin7lw9pC3F5Hseffddzn55JPzXcYxo6/jaWarnXML+1o/m9cIFgE1zrnNzrkE8AyweID1rweezlYx4WCAYMCI69SQiEgP2QyCKcD2jOlaf94hzKwYuAR4rp/lt5vZKjNbVVdXd8QFxUIBXSwWkZz59Kc/zfz583v8fPe73813WYfI5jWCvrrA6+881JXA7/q7NuCcewx4DLxTQ0dUzfrlvBS4kx+2PgLMPaJdiIgMxcMPP5zvEgYlmy2CWmBaxvRUYGc/6y4hi6eFAHCdVHMAOlqy+jIiIqNNNoNgJTDbzGaaWQTvw35Z75XMrAL4APDzLNYCkWIAXLI9qy8jIjLaZO3UkHMuZWZ3ASvwvj76hHNunZnd4S9f6q/6UeDXzrnWbNUCQNgLgkCyLasvIyIy2mT1PgLn3HJgea95S3tNPwk8mc06gO4gQEEgItJDwXQx0d0iSA3tWZ4iIn0pLS3td9mWLVs49dRTc1jN0SmcIPCvEQTTahGIiGQqmL6GCHv9ggdSulgsMqL98gHY/fbw7nPiPLj0qwOucv/99zN9+nTuvPNOAL74xS9iZrzyyiscOHCAZDLJV77yFRYvHui+2EPF43H+5m/+hlWrVhEKhfjmN7/JBz/4QdatW8dtt91GIpGgs7OT5557jsmTJ3PttddSW1tLOp3mC1/4Atddd90Rv+3BKqAg8HoTDKV1akhEDrVkyRLuueee7iD40Y9+xK9+9Ss++9nPUl5ezr59+zjnnHP4yEc+gllft0n1retegrfffpv169dz8cUXs3HjRpYuXcrdd9/NjTfeSCKRIJ1Os3z5ciZPnswvfvELwOvwLhcKJwiCYdIECaXVIhAZ0Q7zl3u2nHHGGezdu5edO3dSV1fHmDFjmDRpEp/97Gd55ZVXCAQC7Nixgz179jBx4sRB7/fVV1/lM5/5DOD1Njp9+nQ2btzIueeeyz/+4z9SW1vLxz72MWbPns28efO49957uf/++7niiit4//vfn62320PhXCMwIxmIEe5Ui0BE+nbNNdfwk5/8hGeffZYlS5bw1FNPUVdXx+rVq1mzZg0TJkzo81kEA+mvY88bbriBZcuWUVRUxIc//GH+53/+hxNPPJHVq1czb948Pv/5z/PlL395ON7WYRVOiwBIBWOE9a0hEenHkiVL+NSnPsW+fft4+eWX+dGPfsT48eMJh8P89re/ZevWrUPe5wUXXMBTTz3Fhz70ITZu3Mi2bds46aST2Lx5M7NmzeJv//Zv2bx5M2+99RZz5syhqqqKm266idLSUp588snhf5N9KLAgKCLqvP69h3KOT0QKwymnnEJzczNTpkxh0qRJ3HjjjVx55ZUsXLiQ+fPnM2fOnCHv88477+SOO+5g3rx5hEIhnnzySaLRKM8++yw//OEPCYfDTJw4kYceeoiVK1dy3333EQgECIfDPProo1l4l4fK2vMIsuVIn0cAsO/rC1jdVMkHHvp194NqRCT/9DyC4TWSnkcw4nQGiyiiQ11Ri4hkKKhTQ53hIoqsgfZkmgrC+S5HREa5t99+m5tvvrnHvGg0yh/+8Ic8VXRkCioICBVTzB7aEql8VyIivYzGa3fz5s1jzZo1+S6jhyM53V9Qp4ZcpJgYCdoSelylyEgSi8Wor68/og8xOcg5R319PbFYbEjbFVSLwMLFFFsH+5MKApGRZOrUqdTW1nI0j6IVTywWY+rUqUPapqCCIBAtJkKHWgQiI0w4HGbmzJn5LqNgFdSpoUCkhCIStHXoGoGISJeCCoJgtISoJWnrSOS7FBGREaOggiAU83ogTcSz+1RMEZHRpKCCIOwHQaq9Oc+ViIiMHAUVBJEi79FyyQ61CEREumQ1CMzsEjPbYGY1ZvZAP+v8hZmtMbN1ZvZyNusJRLwWQadODYmIdMva10fNLAg8DFwE1AIrzWyZc+6djHUqgUeAS5xz28xsfLbqAbofYJ9Wi0BEpFs2WwSLgBrn3GbnXAJ4Buj9sM8bgJ8657YBOOf2ZrEe8FsErkMPsBcR6ZLNIJgCbM+YrvXnZToRGGNmL5nZajO7pa8dmdntZrbKzFYd1Z2HUe8aAYmmI9+HiMgxJptB0FfvUb07EgkBC4DLgQ8DXzCzEw/ZyLnHnHMLnXMLq6urj7yiiBcEgYRODYmIdMlmFxO1wLSM6anAzj7W2eecawVazewV4HRgY1YqipYBEEy2ZGX3IiKjUTZbBCuB2WY208wiwBJgWa91fg6838xCZlYMnA28m7WKuoIgrRaBiEiXrLUInHMpM7sLWAEEgSecc+vM7A5/+VLn3Ltm9ivgLaATeNw5tzZbNRGKkSZAKKUgEBHpktXeR51zy4HlveYt7TX9deDr2ayjmxkdgWKiCgIRkW4FdWcxQCJYQqRTXx8VEelScEGQDJUQ62zPdxkiIiNGwQVBKlRCkWunI6WH04iIQAEGQWe4hDJrp7VDQSAiAgUYBC5SSgnttMT1lDIRESjAICBaTonFae5I5rsSEZERoeCCIBArpVQtAhGRbgUXBMFYGSXEaW5Xi0BEBAowCEJF5YSsk/Y29TckIgIFGASRkgoA4m3qilpEBAowCKLFXhAkWhvzXImIyMhQcEEQLvZ6IE3F1SIQEYECDALzu6JOtTfnuRIRkZGh4IKg65kErl2nhkREoBCDIFbpDTt0akhEBAo4CEIKAhERoCCDoByAcFKnhkREoBCDIBgmbkVEUrpYLCIChRgEQDxURlRBICICFGgQJEJlFKcVBCIikOUgMLNLzGyDmdWY2QN9LP8LM2s0szX+z0PZrKdLMlxBKa0kUp25eDkRkREtlK0dm1kQeBi4CKgFVprZMufcO71W/X/OuSuyVUdf0tEKKthHczzJ2NJoLl9aRGTEyWaLYBFQ45zb7JxLAM8Ai7P4eoMXq6DcWmlQV9QiIlkNginA9ozpWn9eb+ea2Ztm9kszO6WvHZnZ7Wa2ysxW1dXVHXVhVlxJOW00tCkIRESyGQTWxzzXa/oNYLpz7nTg34Cf9bUj59xjzrmFzrmF1dXVR11YqKSKMmunsaXtqPclIjLaZTMIaoFpGdNTgZ2ZKzjnmpxzLf74ciBsZuOyWBMA0dJKAFob92f7pURERrxsBsFKYLaZzTSzCLAEWJa5gplNNDPzxxf59dRnsSYAYmVe1rS3KAhERLL2rSHnXMrM7gJWAEHgCefcOjO7w1++FLgG+BszSwHtwBLnXO/TR8MuVlYFQLJZQSAikrUggO7TPct7zVuaMf5t4NvZrKEvgaJKAJKtCgIRkYK8sxg/CFx7Q17LEBEZCQo0CLxTQ9auFoGISGEGQbEXBOGOA3kuREQk/wozCIJh2gKlxJJqEYiIFGYQAO3hMZSkGvJdhohI3hVsEHRExlCWbiKVVg+kIlLYCjYIUrEqqqyJRnU8JyIFrmCDoLN4HGOtmf2tiXyXIiKSVwUbBKHSasbQzL7mjnyXIiKSVwUbBJGK8YQtTeOBo+/WWkRkNCvYICiuHA9A64E9ea5ERCS/CjgIJgDQ0aggEJHCVrBBECj1uqJONuvUkIgUtoINAor959+07stvHSIieVa4QVDiPfIy2K4gEJHCVrhBEI7RGiynuEOnhkSksBVuEACtkWrKkvXk4KFoIiIjVkEHQSJWTTX7ae5I5bsUEZG8GVQQmNndZlZunv8wszfM7OJsF5dt6dIJjLcDurtYRAraYFsEf+WcawIuBqqB24CvHm4jM7vEzDaYWY2ZPTDAemeZWdrMrhlkPcMiUD6ZahrZ09iey5cVERlRBhsE5g8vA77rnHszY17fG5gFgYeBS4G5wPVmNref9b4GrBhs0cMlVjWFsKXZv29nrl9aRGTEGGwQrDazX+MFwQozKwMO15H/IqDGObfZOZcAngEW97HeZ4DngL2DrGXYlFdPAaClrjbXLy0iMmKEBrneXwPzgc3OuTYzq8I7PTSQKcD2jOla4OzMFcxsCvBR4EPAWYOsZdhEx0wFoOPAjly/tIjIiDHYFsG5wAbnXIOZ3QT8HdB4mG36OnXU+3ua/wLc75xLD7gjs9vNbJWZraqrG8bv/ZdNBKCzaffw7VNEZJQZbBA8CrSZ2enA/wG2At8/zDa1wLSM6alA75PxC4FnzGwLcA3wiJld1XtHzrnHnHMLnXMLq6urB1nyIJR6Hc+FWhUEIlK4BhsEKefddbUY+JZz7ltA2WG2WQnMNrOZZhYBlgDLMldwzs10zs1wzs0AfgLc6Zz72VDewFEJRWkKjaU0vitnLykiMtIM9hpBs5l9HrgZeL//TZ/wQBs451Jmdhfet4GCwBPOuXVmdoe/fOlR1D1sWosmM65hD/Fkmlg4mO9yRERybrBBcB1wA979BLvN7Djg64fbyDm3HFjea16fAeCcu3WQtQyrZNkUpjb+ib1NHRw3tjgfJYiI5NWgTg0553YDTwEVZnYFEHfOHe4awahgldOZbPvYcaA136WIiOTFYLuYuBb4I/Bx4FrgD7m+CzhbisfPJGJp9u3amu9SRETyYrCnhh4EznLO7QUws2rgBbwLvKNaxaTjAWjevRnvS0wiIoVlsN8aCnSFgK9+CNuOaKGq6QAk96tFICKFabAtgl+Z2QrgaX/6OnpdBB61KrxbHQKN2w+zoojIsWlQQeCcu8/MrgbOw7tj+DHn3PNZrSxXIsU0h8ZQ0q5uJkSkMA22RYBz7jm8zuGOOW1FU6hu2E1zPElZbMDbI0REjjkDnuc3s2Yza+rjp9nMmnJVZLalKmcyM7Cbbfvb8l2KiEjODRgEzrky51x5Hz9lzrnyXBWZbaHxJzLV9lG7pz7fpYiI5Nwx8c2fo1U29WQAGnduyHMlIiK5pyAAiifNAaBjl4JARAqPggCgahYAgf01eS5ERCT3FAQAkRIawhMob92C19u2iEjhUBD42stnMs3tZHdTPN+liIjklILAZ+NmM8t2sml3c75LERHJKQWBr/S40yi3dnZt25jvUkREckpB4Cs97gwA4rVv5bkSEZHcUhB0GX8ynRjhunX5rkREJKcUBF2ipTREp1DVspFUujPf1YiI5IyCIEO86mROZBs1dS35LkVEJGeyGgRmdomZbTCzGjN7oI/li83sLTNbY2arzOz8bNZzONGppzPD9vDu1t35LENEJKeyFgRmFgQeBi4F5gLXm9ncXqu9CJzunJsP/BXweLbqGYzKWQsImOPAeyvzWYaISE5ls0WwCKhxzm12ziWAZ4DFmSs451rcwVt5S4C83tYbnHYWAKGdq/NZhohITmUzCKYAmc9/rPXn9WBmHzWz9cAv8FoFhzCz2/1TR6vq6uqyUiwApdXsj0xmQvNa0p3qakJECkM2g8D6mHfIp6tz7nnn3BzgKuAf+tqRc+4x59xC59zC6urq4a2yl9bxZ3Aam9i4R3cYi0hhyGYQ1ALTMqanAjv7W9k59wpwvJmNy2JNh1V6/LlMsv2sW/9uPssQEcmZbAbBSmC2mc00swiwBFiWuYKZnWBm5o+fCUSAvD4mrHL2+wBo3PRaPssQEcmZQT+8fqiccykzuwtYAQSBJ5xz68zsDn/5UuBq4BYzSwLtwHUuz/1A28R5JCxCye6VOOfwc0pE5JiVtSAAcM4tB5b3mrc0Y/xrwNeyWcOQhSLsqzqT+XVvsaOhnaljivNdkYhIVunO4j4Ej/8gcwLbefNd9UQqIsc+BUEfxp12MQD7172Q50pERLJPQdCH4OTTaQ2UUb7zd3p0pYgc8xQEfQkEOTD+bBZ2vsm7O5vyXY2ISFYpCPpRdtoVTLF61r3xar5LERHJKgVBPypOv5I0AWzDL/JdiohIVikI+lMyjp1lp3Nq0yvsb03kuxoRkaxREAwgdMqVzAls57WV6pZaRI5dCoIBTDz7GgA6/vRsnisREckeBcEAbMx0tpadyYKGFTTq9JCIHKMUBIdh829ghu1m1au/zHcpIiJZoSA4jGnnL6GdGG7Nf+a7FBGRrFAQHIZFy9g68WLe1/ZbtmyvzXc5IiLDTkEwCOMvuodi6+DPv3k036WIiAw7BcEgVB2/gHdjZ3DytqeJx+P5LkdEZFgpCAap8+w7mUg9b/7q8XyXIiIyrBQEgzT3A1dTE5zFtLf+DZfSV0lF5NihIBgkCwTZveBeJnfuZtOKpYffQERklFAQDMGii5bwlp1E9epv4uKN+S5HRGRYKAiGIBIOsuPsh6hIN1D7/EP5LkdEZFhkNQjM7BIz22BmNWb2QB/LbzSzt/yf18zs9GzWMxwu/MvLWBa+mMkbvo/b9Wa+yxEROWpZCwIzCwIPA5cCc4HrzWxur9X+DHzAOXca8A/AY9mqZ7hEQgECF/49+10pTf/5SUjq66QiMrpls0WwCKhxzm12ziWAZ4DFmSs4515zzh3wJ18HpmaxnmFzxdlzebTic1Q0b6RjhU4Ricjols0gmAJsz5iu9ef156+BPnt2M7PbzWyVma2qq6sbxhKPTCBgfPTa23gy9WGiq/4d3v3vfJckInLEshkE1sc81+eKZh/EC4L7+1runHvMObfQObewurp6GEs8cvOmVrDlzP/Dm52zSD/3Kdj1Vr5LEhE5ItkMglpgWsb0VGBn75XM7DTgcWCxc64+i/UMu89ddjoPFT9IfbqIzqeXQKM6pROR0SebQbASmG1mM80sAiwBlmWuYGbHAT8FbnbObcxiLVlRFgvz4HUf4raOz5FoaYDvXQlNu/JdlojIkGQtCJxzKeAuYAXwLvAj59w6M7vDzO7wV3sIGAs8YmZrzGxVturJlkUzq7jgggu5of0+ko27vTBQy0BERhFzrs/T9iPWwoUL3apVIysvUulObnnij7D1NX5Q9E2CsVK48ccwcV6+SxMRAcDMVjvnFva1THcWD4NQMMC3bziTbWXzudl9iZQzeOJSeGfZ4TcWEckzBcEwqSqJ8PgnFrI2NZXrO79CsuoE+NHN8Mv7IdWR7/JERPqlIBhGcyaW88StZ/F2czHXdDxEx4Lb4Q9L4bEPQu3qfJcnItInBcEwWzijikdvWsC7eztY/N6VNF71Q2g/AP/xl/Crz4N6LRWREUZBkAUfPGk8T9x6Flvr27jqhTJqb/gtLLgNXn8EvjUfXl8KeriNiIwQCoIsOX/2OH74yUXUt3Sw+PG3WXnq38H/esX7JtGv7odvL4SVj0OyPd+likiBUxBk0YLpVTz/6fMoLwpzw3de59naMXDLz+HG56CkGn7xOfiXefDK16Flb77LFZECpfsIcqCxLcldT7/B/9u0j6vmT+YfrjqVsmgItv4OXv0XqPkNBEJw0mWw4BMw64MQCOa7bBE5hgx0H4GCIEdS6U4eeek9vvXiJiZXxviX685gwfQx3sK6DfDG9+HNp6GtHsomwdzFcMpHYeoiCKjhJiJHR0Ewgqzeup+/fXoNOxvbufmc6dz74ZMoj4W9hakOWP8LWPscbPoNpDugbDLMuQxOuAhmnA/R0vy+AREZlRQEI0xLR4pvrNjA936/hfFlUb5wxVwunzcJs4yeu+NNsHEFrHseNv8Wkm0QCMP0c+H4C2HG+2HSaRAM5++NiMiooSAYodZsb+DzP32bd3c1MX9aJQ9efjJnzag6dMVUB2x7HWpegJoXYe86b364GKYuhOPOhePOgclnQlFlTt+DiIwOCoIRLN3peG51Lf/0mw3saergL0+ewN0Xzmbe1Ir+N2reDdt+74XDtt/D7rfBdXrLxsyESafD5PnecNJ8KO4jXESkoCgIRoH2RJonfvdnlr78Hs3xFBecWM2n/+J4Fs2s6nnKqC/xJtixCnb+CXa9CTvXQMPWg8vLJkH1SVA9xxuO88dLxmb1PYnIyKEgGEWa40l++Po2/uPVzexrSXD6tEpuPmc6V5w2iVh4CF8pbdsPu9/ygmHveqhb7307Kdl6cJ3icTD2eK8VMWYGVM08OF46Hg4XQCIyaigIRqF4Ms2PV23nyde28F5dK5XFYa45cyrXn30cx1cf4TeHOjuhaYcXCHV+OBzYAvv/7M3PfKR0uMQLhMrjoHwyVEyB8ineeNcwXDQM71REckFBMIo55/j95nqeen0bK9btJtXpOG1qBYvnT+HK0ycxviw2PC+UjEPjdi8UDmyBA3/2xhtroanW6zivt6Kqg6FQOt77KRkPpdX+cII3HqtU60IkzxQEx4i9TXGWvbmTn63ZwdodTQQMzjthHBefMpGLTp7AxIphCoW+JNqgeZcfDDu9FkTTjoPjLXXQWgcufei2wYjXpUZJtRcWxWOhaIwXJEWV3sXsrumu8UipwkNkGCkIjkE1e5v5+Zqd/Pdbu/jzPu+8/7wpFVw0dwIXnjyekyeWEwjk+IO0s9NrObTuhZY9fjjs9fpRaq3zh3uh7YC3XqK5/30Fwl4gdAVDrAKi5RAtg1i5N9417DGesVzddIh0UxAcw5xzvFfXwq/f2cML7+zhT9sbcA7GlkR43wnjOP+Esbzv+HFMqyrOd6mHSiUg3uBd2G7f74VDmz9s358xfsB7jkNHk/cNqY4m6Ewdfv+RUi8YomUQKfGmw8X+eHGv6VJ/Xol3fSRznUjJwfWCEbVUZFTKWxCY2SXAt4Ag8Lhz7qu9ls8BvgucCTzonPvG4fapIBjY3uY4r2zcx2s1+3i1Zh97m73HZE4fW8yiGVUsmD6GBdPHcHx1ae5bDMPFOUjFD4ZCvAk6GqGjude8rp9m79RWohUSLd5d2olWf15L36ez+mMBCBVBONbHMOZdQD/sMNr3tsGItyxz2D0eVZ9TclTyEgRmFgQ2AhcBtcBK4Hrn3DsZ64wHpgNXAQcUBMPLOUfN3hZerdnH72rqWb11PwfakgCUx0KcOX0MC44bw/zjKjllcgVVJZE8V5wHzkE64QdDxk+y97QfGsm4F0LJ9kEO45Bq917jaAVCh4ZDMDxAePQzLxjxtguE/GEYgiF/mDEdjPS/rHs6cx+9pgNBtZ5GkIGCIJTF110E1DjnNvtFPAMsBrqDwDm3F9hrZpdnsY6CZWbMnlDG7All3HbeTJxz/HlfK6u3HuCNbQdYvfUAL22o615/ckWMuZMrOHVKOaf4w4nlscPf0Daamfl/oUezewd2Z7pnMPQepjq8sOg9PJJ5yTbvtFu6o9dyf95whNJgHTZAQmBBLzQCoYNDC/jToZ7LLJgxP2Mdy9w+2P+8Qe872GueP24B73Ut2Gter2GP5Zaxr97bjIz/W9kMginA9ozpWuDsI9mRmd0O3A5w3HHHHX1lBcrMmFVdyqzqUj6+cBoAje1J1u1oZO3ORtbtbGLtjkZeXL+HroZiZXGY2eNLOWF8GbPHlzJ7Qimzx5cxoTx6bAfEcAsE/esOJfmuxGsFdaahMwnppHe9JZ3wx5OQTvValuxnOnO9I9zOpf1a0t46nSmvu5TOlBecXfO75nWmeq7vMrbrzFjHpQ92uzKi2SDDw19+1l/D+fcMexXZDIK+PiWO6DyUc+4x4DHwTg0dTVHSU0VRmPedMI73nTCue15rR4r1u5tYu6OJDXuaqdnTwi/X7uJp/7QSQFk0xAkTSpk1rpQZY4s5bmwxM8aWMGNsCRXF6hF1RDPzT/2Eju2bAjs7M4Im1XO8r+DpMT/dK2T8YHGdGcv6mHfIcpcxPpht/GWHbOPvpzI7fwhnMwhqgWkZ01OBnVl8PRkmJdEQC6ZXsWD6wVMlzjnqWxNs2tNCzd5mNu5pYdPeZn5Xs4/n3oj32L6iKOyHQwkzxhYzraqYKZVFTK4sYlJFbGhdZYgcqUAACKir9kHIZhCsBGab2UxgB7AEuCGLrydZZGaMK40yrjTKucf37KyuPZFm2/42tta3srW+jS31rWzb38aa7Qf4xVs76ezVhhtbEmFyZRGTK2NMrixiSmURkyq86fHlMapLo0RC+oaMSK5kLQiccykzuwtYgff10Secc+vM7A5/+VIzmwisAsqBTjO7B5jrnGvKVl0y/IoiQU6aWMZJE8sOWZZIdbK7Mc6OhnZ2Nbazs6GdHQ1xdja0s7mulVc37aM1cejXN8cUhxlfFmN8eZTqsqg3XhZlfHnP8eJINv+WESkMuqFM8so5R1M8xc4GLyT2Nnewt6mDvc1xf9wb1jV3kOrdtACKI0HGFEcYWxqhqsT7GVsSoaok6g8jVJV688aURCiLhnSRWwpSvr4+KnJYZkZFUZiKojAnTyrvd73OTseBtoQXDhkBsb81wf7WBPWtCfa1dLBxdzP1rQk6Un1/YyQSDDCmJMyY4ggVRWEqi8P+MNJdR9dP97KiCGWx0Oi9AU/kMBQEMioEAsbY0ihjS6OcPGngdZ1ztCfT1LckegTF/tYOb9iS4EBbkqb2JFv2tdHQnqCxPUk82f/XDc2gPNYzJMpjYcpiIUqjIcpiYUpjIcpiIcpjIUqj/jJ/Xlk0TCwcUGtERiQFgRxzzIziSIjiqtCQ+liKJ9M0tSdpbE/S0J6ksc0fdv20JQ4ua0+yo6GdlniKlo4UbX1c5+gtFLCD4RD1gqO8V5CURkMUR4KUREIUR71hSde8aIiSSJDiaIjicFAtFBk2CgIRXywcJBYOMr586N15p9KdtHakaYonaelI0RxP0eyPN8VTtGRMe8u86Z0NcX9ekuZ4qs/rIP0pCgcpiQb9oDgYEiWRoDcdDXbPL4l600WREEXhoPcTCRDzx4v9+bFIgEhQLZdCoyAQGQahYICK4sBR30yXSHXSlkjRmkjT2pGi1W9tdA8TKdo60n4rxFuvrcMfJlI0tifZ1dDeY91Eemh32AYMPyiC3UGROV4cCfqhEcwIlcx1A/7QD5dwgGgoSDTkBU/mUK2akUFBIDKCREIBIqEIlcPYa3gi1Ul7Ik1LIkV7IkV7opP2ZNr7SaSJZ4z3GCbTxDPG2xNpGtoS7Grsmu709pdMH3KvyGCFg0YsFCTaFRb+0AuPg4FxcF4fgdJ73X62iYYC/vH1Wj2hoO5V6aIgEDnGdX34ZavrD+cciXQn8V4Bkxk0HalOOlJp4klv2JHq7J7fvTzZSTyVpqNrnWQnB1oT/W5zpOHTJWAQDnrHJuqHQ9ex6pofyVzuT4cz1ouEAkQHtV2QcNAObuPPO7idedsG89NKUhCIyFExM/+v7iAV5KY7B+ccqU53aKAkewZOZogkun7SfYxnzvOnk+lOOlKdNMdT1PdaJ+mPd/jD4RQMWI9gCAcDhEPe9PVnHcenLpg1rK8HCgIRGYXMDn5Ylkbz+zHmnCOZdt1B0R0SfQRPV7j0DJ40qU5v+2TKkfRDqCuMuuYl0p1Ul0Wz8h4UBCIiR8HMiIS80z5k53M663S1RESkwCkIREQKnIJARKTAKQhERAqcgkBEpMApCERECpyCQESkwCkIREQK3Kh7VKWZ1QFbj3DzccC+YSxnuIzUumDk1qa6hkZ1Dc2xWNd051x1XwtGXRAcDTNb1d8zO/NppNYFI7c21TU0qmtoCq0unRoSESlwCgIRkQJXaEHwWL4L6MdIrQtGbm2qa2hU19AUVF0FdY1AREQOVWgtAhER6UVBICJS4AomCMzsEjPbYGY1ZvZAjl97mpn91szeNbN1Zna3P/+LZrbDzNb4P5dlbPN5v9YNZvbhLNa2xcze9l9/lT+vysx+Y2ab/OGYXNZlZidlHJM1ZtZkZvfk43iZ2RNmttfM1mbMG/LxMbMF/nGuMbN/NbOjejBtP3V93czWm9lbZva8mVX682eYWXvGcVua47qG/HvLUV3PZtS0xczW+PNzebz6+2zI7b8x59wx/wMEgfeAWUAEeBOYm8PXnwSc6Y+XARuBucAXgXv7WH+uX2MUmOnXHsxSbVuAcb3m/V/gAX/8AeBrua6r1+9uNzA9H8cLuAA4E1h7NMcH+CNwLmDAL4FLs1DXxUDIH/9aRl0zMtfrtZ9c1DXk31su6uq1/J+Ah/JwvPr7bMjpv7FCaREsAmqcc5udcwngGWBxrl7cObfLOfeGP94MvAtMGWCTxcAzzrkO59yfgRq895Ari4Hv+ePfA67KY10XAu855wa6mzxrdTnnXgH29/F6gz4+ZjYJKHfO/d55/2O/n7HNsNXlnPu1cy7lT74OTB1oH7mqawB5PV5d/L+crwWeHmgfWaqrv8+GnP4bK5QgmAJsz5iuZeAP4qwxsxnAGcAf/Fl3+U35JzKaf7ms1wG/NrPVZna7P2+Cc24XeP9QgfF5qKvLEnr+B8338YKhH58p/niu6gP4K7y/CrvMNLM/mdnLZvZ+f14u6xrK7y3Xx+v9wB7n3KaMeTk/Xr0+G3L6b6xQgqCvc2U5/96smZUCzwH3OOeagEeB44H5wC685inktt7znHNnApcCnzazCwZYN6fH0cwiwEeAH/uzRsLxGkh/deT6uD0IpICn/Fm7gOOcc2cA/xv4TzMrz2FdQ/295fr3eT09/9jI+fHq47Oh31X7qeGoaiuUIKgFpmVMTwV25rIAMwvj/aKfcs79FMA5t8c5l3bOdQLf4eDpjJzV65zb6Q/3As/7Nezxm5pdzeG9ua7LdynwhnNuj19j3o+Xb6jHp5aep2myVp+ZfQK4ArjRP0WAfxqh3h9fjXde+cRc1XUEv7dcHq8Q8DHg2Yx6c3q8+vpsIMf/xgolCFYCs81spv9X5hJgWa5e3D8H+R/Au865b2bMn5Sx2keBrm80LAOWmFnUzGYCs/EuBA13XSVmVtY1jnexca3/+p/wV/sE8PNc1pWhx19q+T5eGYZ0fPymfbOZneP/W7glY5thY2aXAPcDH3HOtWXMrzazoD8+y69rcw7rGtLvLVd1+f4SWO+c6z6tksvj1d9nA7n+N3Y0V7xH0w9wGd4V+feAB3P82ufjNdPeAtb4P5cBPwDe9ucvAyZlbPOgX+sGjvKbCQPUNQvvGwhvAuu6jgswFngR2OQPq3JZl/86xUA9UJExL+fHCy+IdgFJvL+6/vpIjg+wEO8D8D3g2/h39Q9zXTV454+7/o0t9de92v/9vgm8AVyZ47qG/HvLRV3+/CeBO3qtm8vj1d9nQ07/jamLCRGRAlcop4ZERKQfCgIRkQKnIBARKXAKAhGRAqcgEBEpcAoCkRwys78ws//Odx0imRQEIiIFTkEg0gczu8nM/uj3R//vZhY0sxYz+ycze8PMXjSzan/d+Wb2uh18DsAYf/4JZvaCmb3pb3O8v/tSM/uJec8OeGpI/caLZIGCQKQXMzsZuA6vQ775QBq4ESjB6/voTOBl4O/9Tb4P3O+cOw3vDtqu+U8BDzvnTgfeh3dnK3g9TN6D17f8LOC8LL8lkQGF8l2AyAh0IbAAWOn/sV6E1+lXJwc7J/sh8FMzqwAqnXMv+/O/B/zY78NpinPueQDnXBzA398fnd+3jXlPxZoBvJr1dyXSDwWByKEM+J5z7vM9Zpp9odd6A/XPMtDpno6M8TT6fyh5plNDIod6EbjGzMZD9/Njp+P9f7nGX+cG4FXnXCNwIOPhJTcDLzuvT/laM7vK30fUzIpz+SZEBkt/iYj04px7x8z+Du/JbQG8His/DbQCp5jZaqAR7zoCeN0EL/U/6DcDt/nzbwb+3cy+7O/j4zl8GyKDpt5HRQbJzFqcc6X5rkNkuOnUkIhIgVOLQESkwKlFICJS4BQEIiIFTkEgIlLgFAQiIgVOQSAiUuD+P8qwgQeSWmnOAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(random_init_net.losses)\n",
    "plt.plot(random_init_net.val_losses)\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train_loss', 'val_loss'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 파이썬의 제너레이터 함수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo()\n",
      "10\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "ret=0\n",
    "\n",
    "def foo():\n",
    "    ret = 10   # 스택\n",
    "    print('foo()')\n",
    "    return ret\n",
    "    \n",
    "c = foo()\n",
    "print(c)\n",
    "print(ret)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<generator object foo at 0x000002B64F4EEC10>\n",
      "foo()\n",
      "10\n"
     ]
    }
   ],
   "source": [
    "def foo():\n",
    "    print('foo()')\n",
    "    yield 10\n",
    "#     return 10\n",
    "\n",
    "c = foo()\n",
    "print(c)\n",
    "print(next(c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo()\n",
      "10\n",
      "20\n"
     ]
    }
   ],
   "source": [
    "def foo():\n",
    "    print('foo()')\n",
    "    yield 10\n",
    "    yield 20\n",
    "\n",
    "for data in foo():\n",
    "    print(data)\n",
    "    \n",
    "# c = foo()\n",
    "# print(c)\n",
    "# n = next(c)\n",
    "# print(n)\n",
    "# n = next(c)\n",
    "# print(n)\n",
    "# n = next(c)\n",
    "# print(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo()\n",
      "(100,)\n",
      "(10,)\n",
      "[0 1 2 3 4 5 6 7 8 9]\n"
     ]
    }
   ],
   "source": [
    "def foo():\n",
    "    print('foo()')\n",
    "    a = np.arange(100)\n",
    "    print(a.shape)\n",
    "    return a[0:10]\n",
    "\n",
    "c = foo()\n",
    "print(c.shape)\n",
    "print(c)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo()\n",
      "[0 1 2 3 4 5 6 7 8 9]\n",
      "foo()\n",
      "[10 11 12 13 14 15 16 17 18 19]\n",
      "foo()\n",
      "[20 21 22 23 24 25 26 27 28 29]\n"
     ]
    }
   ],
   "source": [
    "start = 0\n",
    "end   = 0\n",
    "i = 0\n",
    "batch = 10\n",
    "\n",
    "def foo():\n",
    "    global i\n",
    "    print('foo()')\n",
    "    a = np.arange(100)\n",
    "#     print(a.shape)\n",
    "    start = i * batch\n",
    "    end   = (i+1)*batch\n",
    "    i += 1\n",
    "    return a[start:end]\n",
    "\n",
    "print(foo())\n",
    "print(foo())\n",
    "print(foo())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foo()\n",
      "[0 1 2 3 4 5 6 7 8 9]\n",
      "[10 11 12 13 14 15 16 17 18 19]\n",
      "[20 21 22 23 24 25 26 27 28 29]\n",
      "[30 31 32 33 34 35 36 37 38 39]\n",
      "[40 41 42 43 44 45 46 47 48 49]\n",
      "[50 51 52 53 54 55 56 57 58 59]\n",
      "[60 61 62 63 64 65 66 67 68 69]\n",
      "[70 71 72 73 74 75 76 77 78 79]\n",
      "[80 81 82 83 84 85 86 87 88 89]\n",
      "[90 91 92 93 94 95 96 97 98 99]\n",
      "[100 101 102 103]\n"
     ]
    }
   ],
   "source": [
    "def foo():\n",
    "    print('foo()')\n",
    "    a = np.arange(104)\n",
    "    batch = 10\n",
    "    length = len(a)\n",
    "    bins = length // batch\n",
    "    if length % batch != 0 :\n",
    "        bins += 1\n",
    "    for i in range(bins):\n",
    "        start = batch*i\n",
    "        end = batch*(i+1)\n",
    "        yield a[start:end]\n",
    "\n",
    "for n in foo():\n",
    "    print(n)\n",
    "# c = foo()\n",
    "# print(c)\n",
    "# print(next(c))\n",
    "# print(next(c))\n",
    "# print(next(c))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<generator object foo at 0x000002B64F6B6D60>\n",
      "foo()\n",
      "10\n",
      "20\n",
      "30\n",
      "40\n",
      "50\n",
      "60\n",
      "70\n",
      "80\n",
      "90\n",
      "100\n"
     ]
    }
   ],
   "source": [
    "def foo():\n",
    "    print('foo()')\n",
    "    for i in range(10):\n",
    "        yield  (i+1)*10\n",
    "    \n",
    "c = foo()\n",
    "print(c)\n",
    "for ret in c:\n",
    "    print(ret)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1 2 3 4]\n",
      "[5 6 7 8]\n",
      "[ 9 10]\n"
     ]
    }
   ],
   "source": [
    "a = np.array([1,2,3,4,5,6,7,8,9,10])\n",
    "print(a[0:4])\n",
    "print(a[4:8])\n",
    "print(a[8:100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 미니 배치 학습의 구현"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MinibatchNetwork(RandomInitNetwork):\n",
    "    \n",
    "    def __init__(self, units=10, batch_size=32, learning_rate=0.1, l1=0, l2=0):\n",
    "        super().__init__(units, learning_rate, l1, l2)\n",
    "        self.batch_size = batch_size     # 배치 크기\n",
    "        \n",
    "    def fit(self, x, y, epochs=100, x_val=None, y_val=None):\n",
    "        y_val = y_val.reshape(-1, 1)     # 타깃을 열 벡터로 바꿉니다.\n",
    "        self.init_weights(x.shape[1])    # 은닉층과 출력층의 가중치를 초기화합니다.\n",
    "        np.random.seed(42)\n",
    "        # epochs만큼 반복합니다.\n",
    "        for i in range(epochs):\n",
    "            loss = 0\n",
    "            # 제너레이터 함수에서 반환한 미니배치를 순환합니다.\n",
    "            for x_batch, y_batch in self.gen_batch(x, y):\n",
    "                y_batch = y_batch.reshape(-1, 1) # 타깃을 열 벡터로 바꿉니다.\n",
    "                m = len(x_batch)                 # 샘플 개수를 저장합니다.\n",
    "                a = self.training(x_batch, y_batch, m)\n",
    "                # 안전한 로그 계산을 위해 클리핑합니다.\n",
    "                a = np.clip(a, 1e-10, 1-1e-10)\n",
    "                # 로그 손실과 규제 손실을 더하여 리스트에 추가합니다.\n",
    "                loss += np.sum(-(y_batch*np.log(a) + (1-y_batch)*np.log(1-a)))\n",
    "            self.losses.append((loss + self.reg_loss()) / len(x))\n",
    "            # 검증 세트에 대한 손실을 계산합니다.\n",
    "            self.update_val_loss(x_val, y_val)\n",
    "\n",
    "    # 미니배치 제너레이터 함수\n",
    "    def gen_batch(self, x, y):\n",
    "        length = len(x)\n",
    "        bins = length // self.batch_size # 미니배치 횟수\n",
    "        if length % self.batch_size:\n",
    "            bins += 1                    # 나누어 떨어지지 않을 때\n",
    "        indexes = np.random.permutation(np.arange(len(x))) # 인덱스를 섞습니다.\n",
    "        x = x[indexes]\n",
    "        y = y[indexes]\n",
    "        for i in range(bins):\n",
    "            start = self.batch_size * i\n",
    "            end = self.batch_size * (i + 1)\n",
    "            yield x[start:end], y[start:end]   # batch_size만큼 슬라이싱하여 반환합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.978021978021978"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "minibatch_net = MinibatchNetwork(l2=0.01, batch_size=32)\n",
    "minibatch_net.fit(x_train_scaled, y_train, \n",
    "                  x_val=x_val_scaled, y_val=y_val, epochs=500)\n",
    "minibatch_net.score(x_val_scaled, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAppElEQVR4nO3deXidZZ3/8ff37Nm6p/sWbEsFCgVKgQFEZcCySBEUKpu4MdVBUX8idXQcdfAa/Mm4zIj0h4rMKAMiCFStVkWhg2xtsUAL3SitCaVtmtI2e85y//54nqQn6Wmatnlykjyf13Wd6zzbec73Tq/mk/t+NnPOISIi4RUpdgEiIlJcCgIRkZBTEIiIhJyCQEQk5BQEIiIhFyt2AYdr1KhRburUqcUuQ0RkQFm1atUu51xloXUDLgimTp3KypUri12GiMiAYmZbD7ZOQ0MiIiGnIBARCTkFgYhIyA24YwQiMvik02lqampoaWkpdikDXiqVYuLEicTj8R5/RkEgIkVXU1NDRUUFU6dOxcyKXc6A5Zyjrq6Ompoaqqqqevw5DQ2JSNG1tLQwcuRIhcBRMjNGjhx52D0rBYGI9AsKgd5xJD/H0ATB+u31/Pvv11PX0FrsUkRE+pXQBMGmnQ385582UdfYVuxSRET6ldAEQTTidZcyWT2IR0Q627NnDz/4wQ8O+3MXXXQRe/bsOezP3XDDDTz00EOH/bmghC4IsjkFgYh0drAgyGaz3X5u6dKlDBs2LKCq+k5oTh+NtQeBHs0p0q997VdreWXbvl7d53Hjh/Av7z3+oOsXLVrEa6+9xuzZs4nH45SXlzNu3DhWr17NK6+8wmWXXUZ1dTUtLS3cfPPN3HjjjcD+e581NDRw4YUXcvbZZ/P0008zYcIEHnvsMUpKSg5Z2+OPP87nP/95MpkMp512GnfddRfJZJJFixaxZMkSYrEYF1xwAXfccQe/+MUv+NrXvkY0GmXo0KEsX768V34+oQmCSEePIFfkSkSkv7n99ttZs2YNq1ev5oknnuDiiy9mzZo1Hefi33PPPYwYMYLm5mZOO+00rrjiCkaOHNlpHxs3buT+++/nhz/8IVdeeSUPP/ww1157bbff29LSwg033MDjjz/OjBkzuP7667nrrru4/vrreeSRR1i3bh1m1jH89PWvf51ly5YxYcKEIxqSOpjQBEFHj0A5INKvdfeXe1+ZO3dupwuy/uM//oNHHnkEgOrqajZu3HhAEFRVVTF79mwATj31VLZs2XLI71m/fj1VVVXMmDEDgA996EPceeed3HTTTaRSKT72sY9x8cUXc8kllwBw1llnccMNN3DllVdy+eWX90JLPaE7RpBRj0BEDqGsrKxj+oknnuCPf/wjzzzzDC+++CInn3xywQu2kslkx3Q0GiWTyRzye9xBhqpjsRjPP/88V1xxBY8++ijz5s0DYPHixdx2221UV1cze/Zs6urqDrdphb+vV/YyAOhgsYgcTEVFBfX19QXX7d27l+HDh1NaWsq6det49tlne+17Z86cyZYtW9i0aRPTpk3jpz/9Keeeey4NDQ00NTVx0UUXccYZZzBt2jQAXnvtNU4//XROP/10fvWrX1FdXX1Az+RIKAhEJPRGjhzJWWedxQknnEBJSQljxozpWDdv3jwWL17MiSeeyLHHHssZZ5zRa9+bSqX4yU9+wgc+8IGOg8ULFy5k9+7dzJ8/n5aWFpxzfOc73wHglltuYePGjTjnOO+88zjppJN6pQ47WNekv5ozZ447kieUvVi9h/l3/oUff2gO5719zKE/ICJ95tVXX+Xtb397scsYNAr9PM1slXNuTqHtQ3iMYGAFn4hI0EIzNBSLekGQUxCISB/5x3/8R/7yl790WnbzzTfz4Q9/uEgVFRaaIIiaegQi0rfuvPPOYpfQI6EbGsoNsGMiIiJBC00QxCJeU3XTORGRzkITBH4O6PRREZEuAg0CM5tnZuvNbJOZLTrINu80s9VmttbMngyqlvYegW46JyLSWWBBYGZR4E7gQuA44INmdlyXbYYBPwAudc4dD3wgqHraewQ6WCwivaG8vPyg67Zs2cIJJ5zQh9UcnSB7BHOBTc65zc65NuABYH6Xba4Gfumc+xuAc25nUMW09wh0+qiISGdBnj46AajOm68BTu+yzQwgbmZPABXA95xz/911R2Z2I3AjwOTJk4+oGF1QJjJA/HYRbH+5d/c5dhZceHu3m9x6661MmTKFT37ykwB89atfxcxYvnw5b731Ful0mttuu43587v+Pdu9lpYWPvGJT7By5UpisRjf/va3ede73sXatWv58Ic/TFtbG7lcjocffpjx48dz5ZVXUlNTQzab5Z//+Z+56qqrjrjZPRVkEFiBZV1/C8eAU4HzgBLgGTN71jm3odOHnLsbuBu8W0wcSTFRPY9ARLqxYMECPvOZz3QEwYMPPsjvfvc7PvvZzzJkyBB27drFGWecwaWXXopZoV9vhbVfS/Dyyy+zbt06LrjgAjZs2MDixYu5+eabueaaa2hrayObzbJ06VLGjx/Pb37zG8C74V1fCDIIaoBJefMTgW0FttnlnGsEGs1sOXASsIFepucRiAwQh/jLPSgnn3wyO3fuZNu2bdTW1jJ8+HDGjRvHZz/7WZYvX04kEuGNN95gx44djB07tsf7feqpp/jUpz4FeHcbnTJlChs2bODMM8/kG9/4BjU1NVx++eVMnz6dWbNm8fnPf55bb72VSy65hHPOOSeo5nYS5DGCFcB0M6syswSwAFjSZZvHgHPMLGZmpXhDR68GUUzE1CMQke69//3v56GHHuLnP/85CxYs4L777qO2tpZVq1axevVqxowZU/BZBN052I09r776apYsWUJJSQnvec97+NOf/sSMGTNYtWoVs2bN4otf/CJf//rXe6NZhxRYj8A5lzGzm4BlQBS4xzm31swW+usXO+deNbPfAS8BOeBHzrk1QdQT0zECETmEBQsW8PGPf5xdu3bx5JNP8uCDDzJ69Gji8Th//vOf2bp162Hv8x3veAf33Xcf7373u9mwYQN/+9vfOPbYY9m8eTPHHHMMn/70p9m8eTMvvfQSM2fOZMSIEVx77bWUl5dz77339n4jCwj0XkPOuaXA0i7LFneZ/xbwrSDrAO+ZxWY6a0hEDu7444+nvr6eCRMmMG7cOK655hre+973MmfOHGbPns3MmTMPe5+f/OQnWbhwIbNmzSIWi3HvvfeSTCb5+c9/zs9+9jPi8Thjx47lK1/5CitWrOCWW24hEokQj8e56667AmjlgULzPAKAaf+0lBvfcQxfmHf4/5giEhw9j6B36XkE3YhGTFcWi4h0EZrbUIN3nCCrm86JSC95+eWXue666zotSyaTPPfcc0Wq6MiEKggiEdPBYpF+yjl3WOfn9wezZs1i9erVxS6jkyMZ7g/V0FAsYnoegUg/lEqlqKurO6JfYrKfc466ujpSqdRhfS5UPYKoegQi/dLEiROpqamhtra22KUMeKlUiokTJx7WZ8ITBBuW8VjmJv6n+bvArGJXIyJ54vE4VVVVxS4jtMIzNJRpYQI7iWRbi12JiEi/Ep4giMS992y6uHWIiPQz4QmCqBcEllMQiIjkC08QRPzDIblMcesQEelnwhMEUQ0NiYgUEp4giLQPDalHICKSLzxBEPWGhnSMQESks/AEQXuPwKlHICKSLzxBoLOGREQKCk8Q6BiBiEhB4QkC/xhBREEgItJJeIIgoqEhEZFCwhME7ccInIJARCRfeILAv7LYshoaEhHJF54giCYA9QhERLoKURDorCERkUICDQIzm2dm681sk5ktKrD+nWa218xW+6+vBFaMTh8VESkosCeUmVkUuBM4H6gBVpjZEufcK102/V/n3CVB1dEhEiFHRKePioh0EWSPYC6wyTm32TnXBjwAzA/w+w4pZzEdIxAR6SLIIJgAVOfN1/jLujrTzF40s9+a2fGFdmRmN5rZSjNbeTQPt85ajIjLHvHnRUQGoyCDwAosc13mXwCmOOdOAv4TeLTQjpxzdzvn5jjn5lRWVh5xQTmLEdEFZSIinQQZBDXApLz5icC2/A2cc/uccw3+9FIgbmajgiooF4kRdVmc65pHIiLhFWQQrACmm1mVmSWABcCS/A3MbKyZmT8916+nLqiCnMWIkSWdVRCIiLQL7Kwh51zGzG4ClgFR4B7n3FozW+ivXwy8H/iEmWWAZmCBC/DP9VwkRswyZHI5EiG6hEJEpDuBBQF0DPcs7bJscd7094HvB1lDp++OxIiTJZ1xkOirbxUR6d9C9Wdx+9BQWzZX7FJERPqNcAVBJE6cLJmcgkBEpF3IgiBGjIw3NCQiIkDIgoBoXENDIiJdhCsIIjHipqEhEZF8oQoCF00Q19CQiEgnoQoConHiZDQ0JCKSJ2RBkCJBhoyCQESkQ7iCIJYkSZtuMSEikidkQZAgYRnS6hGIiHQIWRCkSJLWMQIRkTyB3muov4nEkkRJk9HQkIhIh1D1CCzu9Qg0NCQisl+4giCWJGkZ2jJ6XKWISLtQBUE0ngQgk24tciUiIv1HuIIgUQJApq25yJWIiPQfoQqCWCIFQKa1pciViIj0H6EMgmxaQSAi0i5UQWCx9iDQ0JCISLtQBQEx72Bxrk0Hi0VE2oUzCDQ0JCLSIVxBEE0AkNPpoyIiHQINAjObZ2brzWyTmS3qZrvTzCxrZu8Psh78YwRk1CMQEWkXWBCYWRS4E7gQOA74oJkdd5DtvgksC6qWDh1DQ+oRiIi0C7JHMBfY5Jzb7JxrAx4A5hfY7lPAw8DOAGvx+EFAVkEgItIuyCCYAFTnzdf4yzqY2QTgfcDi7nZkZjea2UozW1lbW3vkFUX9IMgoCERE2gUZBFZgWdf7P38XuNU51+1d4Jxzdzvn5jjn5lRWVh55RTHvYLGCQERkvyCfR1ADTMqbnwhs67LNHOABMwMYBVxkZhnn3KOBVBQvBSCS1cFiEZF2QQbBCmC6mVUBbwALgKvzN3DOVbVPm9m9wK8DCwGAuHfTuWhOQSAi0i6wIHDOZczsJryzgaLAPc65tWa20F/f7XGBQMS8IIipRyAi0iHQR1U655YCS7ssKxgAzrkbgqwFgGiMjMUVBCIiecJ1ZTGQjqSI53SwWESkXeiCIBtNkXAtOKcH2IuIQAiDIBNNkaKV1oweYC8iAiEMgly0hBLaaGrTA+xFRCCMQRAroYRWmtoyxS5FRKRf6FEQmNnNZjbEPD82sxfM7IKgiwuCi5eQsjaa1SMQEQF63iP4iHNuH3ABUAl8GLg9sKoCZPH2HoGCQEQEeh4E7fcNugj4iXPuRQrfS6j/i5eSIq0gEBHx9TQIVpnZ7/GCYJmZVQAD8rSbSKKUEtMxAhGRdj29svijwGxgs3OuycxG4A0PDTiRRCkpnTUkItKhpz2CM4H1zrk9ZnYt8GVgb3BlBSeaLKWEVh0sFhHx9TQI7gKazOwk4AvAVuC/A6sqQLFkmXcdQWu62KWIiPQLPQ2CjPPuyTAf+J5z7ntARXBlBSdWUk7EHG0tDcUuRUSkX+jpMYJ6M/sicB1wjv/A+XhwZQUnVjIEgGxzfZErERHpH3raI7gKaMW7nmA73rOHvxVYVQGypB8ELQoCERHoYRD4v/zvA4aa2SVAi3NuQB4jIFkOgGvZV+RCRET6h57eYuJK4HngA8CVwHNm9v4gCwtMwguCbKt6BCIi0PNjBF8CTnPO7QQws0rgj8BDQRUWGL9HgIaGRESAnh8jiLSHgK/uMD7bv/jHCGjTWUMiItDzHsHvzGwZcL8/fxVdnkU8YPhDQ9amHoGICPQwCJxzt5jZFcBZeDebu9s590iglQXFHxqKphuLXIiISP/Q0x4BzrmHgYcDrKVvxEvJESGWURCIiMAhxvnNrN7M9hV41ZvZIc+/NLN5ZrbezDaZ2aIC6+eb2UtmttrMVprZ2UfTmB4xIx0tJZltJJvTA+xFRLrtETjnjvg2Ev7Vx3cC5wM1wAozW+KceyVvs8eBJc45Z2YnAg8CM4/0O3sqHSunvLWZhpYMQ0sH5AXSIiK9Jsgzf+YCm5xzm51zbcADePcq6uCca/DvYQRQBvTJn+jp5FCGWiP7WnTjORGRIINgAlCdN1/jL+vEzN5nZuuA3wAfCbCeDrnkMIZaI/UtejiNiEiQQVDoUZYH/MXvnHvEOTcTuAz414I7MrvRP4awsra29ugrSw1jGA3sbVaPQEQkyCCoASblzU8Eth1sY+fccuBtZjaqwLq7nXNznHNzKisrj7qwSNkIhlkDe5vbjnpfIiIDXZBBsAKYbmZVZpYAFgBL8jcws2lmZv70KUAC76rlQMXLRzCMRt5qVBCIiPT4OoLD5ZzLmNlNwDIgCtzjnFtrZgv99YuBK4DrzSwNNANX5R08DkyyYhRxS1PfoDuQiogEFgQAzrmldLkVhR8A7dPfBL4ZZA2FxMtHANBav7uvv1pEpN8ZmDeOO1olwwFI1+8qciEiIsUX0iDwegS5xsAPR4iI9HvhDILyMQDEmnrhVFQRkQEupEHgnYKaaNXQkIhIOIMgNYyMxUm1KAhERMIZBGY0J0YyJLeHxlbdZkJEwi2cQQBkSkZSyR521rcWuxQRkaIKbRC4sjFU2l527GspdikiIkUV2iCIDpvAWKtTEIhI6IU2CJIjpzDCGnhrz55ilyIiUlQhDoLJALTs2lrkSkREiiu0QWDDvDtk5/bUFLkSEZHiCm0QMNQLgmi9gkBEwi28QVAxjhwRSpreLHYlIiJFFd4giMaoj1cypG07ffAIBBGRfiu8QQA0lY5jTG4X+/QQexEJsVAHQW7IRMbbLmreaip2KSIiRRPqIIiPmMw4q6O6rrHYpYiIFE2og6B87DQSlmXPm68VuxQRkaIJdRCUTDgBgOz2tUWuRESkeEIdBDb67QDE69YVuRIRkeIJdRCQGsLu2BiG1m8qdiUiIkUT7iAA6odMY3JmK3ub08UuRUSkKAINAjObZ2brzWyTmS0qsP4aM3vJfz1tZicFWU8hbvRxHGPb2LT9rb7+ahGRfiGwIDCzKHAncCFwHPBBMzuuy2avA+c6504E/hW4O6h6DqZi0gkkLcObr7/S118tItIvBNkjmAtscs5tds61AQ8A8/M3cM497Zxr/1P8WWBigPUUNPyYUwFIb13Z118tItIvBBkEE4DqvPkaf9nBfBT4baEVZnajma00s5W1tbW9WCJExhzPPqtgeO1zvbpfEZGBIsggsALLCt7dzczehRcEtxZa75y72zk3xzk3p7KyshdLBCIRtpbPZnrTat18TkRCKcggqAEm5c1PBLZ13cjMTgR+BMx3ztUFWM9BtU48iwnspHqzricQkfAJMghWANPNrMrMEsACYEn+BmY2GfglcJ1zbkOAtXRr1KzzANj+0h+LVYKISNEEFgTOuQxwE7AMeBV40Dm31swWmtlCf7OvACOBH5jZajMryhHbyceeSi3DSG75UzG+XkSkqGJB7tw5txRY2mXZ4rzpjwEfC7KGnohEo7xSfiZz9j4BmVaIJYtdkohInwn9lcXt9lRdQhnNNPz1l8UuRUSkTykIfFNOu4jXc2NofabPr2kTESkqBYHvxInDeSQ6j5G7X4DtLxe7HBGRPqMg8EUixu4Z76eFOLkVPy52OSIifUZBkOeM46exJPN3uNX3Q/2OYpcjItInFAR5zp1RyY/tMsim4alvF7scEZE+oSDIU5GKc/ysU3jUnYtbeQ/s1JXGIjL4KQi6uGrOJG5vvYLWWAU8eB201he7JBGRQCkIuphbNYKykRP4VvkXoG4TLPkU6GZ0IjKIKQi6MDM+MGcSP35jEnVnLIK1j8Bziw/9QRGRAUpBUMCVcyaRiEX498YLYeYl8Psvw0sPFrssEZFAKAgKqKxIcsUpE3johTeoO/+7MPlM+OXH4YnbNUwkIoOOguAgPnbOMWSyOb731A649pcw+xp44t+8QEg3F7s8EZFeoyA4iLdVlnPdGVP42bNbWbOjGebfCef9C7z8C/jR+bBrU7FLFBHpFQqCbnzugmMZUZbky4+uIeeAcz4H1zwE+96A//cOWP4t9Q5EZMBTEHRjaEmcf7poJqur9/Dgympv4fTzYeFTMO3d8Kfb4PunwcsP6diBiAxYCoJDeN/JEzi9agS3/eZVNu1s8BYOnQBX/Qw+9GsoGQYPfxTueQ/UrCpqrSIiR0JBcAhmxneumk0yFuEffrqS+pb0/pVV58CNT8Kl/wm7X4cfvRt+ejmsvh9a9hWvaBGRw6Ag6IHxw0q485pT2FLXxM0PrCaTze1fGYnCKdfDp1bBuYtg10Z4dCHcMR0evB5e/RWkW4pXvIjIIZgbYGPbc+bMcStXFuUZ9/zs2a18+dE1fHDuJL5x2SwiETtwI+eg+nlY8xCs+SU07YLkEJh2HrztPJh5MZSO6PviRSTUzGyVc25OwXUKgsNzx7L1fP/Pm7hyzkT+7fITiRYKg3bZDLz+JKx5GDY/CftqvOWjZnhXLE84BSadAeWVfVO8iIRWd0EQ6+tiBrr/c8EMohHje49vpDmd49tXnkQ8epARtmjM6wlMO8/rKdSshC3/Cxv/0Pl5B+VjYMzxMOYEGD8bxsyCEVUQjfdJm0Qk3ALtEZjZPOB7QBT4kXPu9i7rZwI/AU4BvuScu+NQ+yx2j6Dd4idf4/bfruO0qcO58+pTGD0kdXg7SLfAtr/Cthdg+xrY8TLUrodsm7c+EoPhVV7vYdQ0733kdBj5NigdCdZNT0REpIuiDA2ZWRTYAJwP1AArgA86517J22Y0MAW4DHhrIAUBwJIXt3HrQy9RkYrx/atPYW7VUY79Z9pg5ytQuw52bfBfG6HuNcjlna0UK/FOYR060X9NgiH+/JDxUFYJJcMVFiLSoVhDQ3OBTc65zX4RDwDzgY4gcM7tBHaa2cUB1hGYS08az4wx5Sz86SquuvsZrj9jCrfMm0l58gh/rLGENzQ0fnbn5dkM7NnqPR9h92bYW7P/telxqN8OdAn0SNwLhPJKKBsN5aOhbJQ33bHcf6WGQfwwezQiMmgEGQQTgOq8+Rrg9AC/ryhmjh3Crz99DncsW89/PbOF37+yg1vecyzzZ0/o/kDy4YjGvCGhkW8rvD7TBvXbvGDY9yY07oTGWmio9aYbdno9jcba/UNPXcVSXiCUV3q9i9Qw72K51FBvOlnhzZeP8U6ZTZRDosx7xcu8EBORASnIICj0W/CIxqHM7EbgRoDJkycfTU2BKE/G+Oqlx/Pek8bzlcfW8LkHX+Tu5ZtZdOFMzp1RiQU9RBNLwPCp3qs7zkHrvv0B0VjrvZr3QMse771hh3cvpZ2vQPNeaN3bsxoicUiUeqGQKN0fEIlSiJf6wdE+3R4gpXnv/jadlvnbRaJH9eMRke4FGQQ1wKS8+YnAtiPZkXPubuBu8I4RHH1pwTh1ynB+ddPZ/PrlN7lj2Xpu+MkKTps6nI+cVcX5x40hdrCzi/qKmf8X/lDvAHRP5LJeeLTWewHS8pY3VNXWAG2N/nsTpBu997bG/dPpJmjaDW3V3nRbo/eeOcwL7KJJr0eSGuIdH4nGIZaEaCLvVWBZLNFlm7xtDzYdSxZYn4RkuRdUCiUZhIIMghXAdDOrAt4AFgBXB/h9/UIkYlx60njmHT+W+5//Gz/838184r4XGD80xXVnTuUDcyYyqjxZ7DJ7LhL1DjyXDIdhvdQby2X3h0LHe5MXKu3TXYOltR5a9kKm1RveyrZ5Q2LpvZBthWzaX5f25/312TZw2d6pG/yASe1/xVNeeMRKvPd4Sef5aMI/aG9gEe/naRHvrLBIFCya9+4vP+Sy6IGfjca874yX7A+wSGz/62DzOqFACP700YuA7+KdPnqPc+4bZrYQwDm32MzGAiuBIUAOaACOc84d9EY9/emsoZ7I5hyPv7qDe5/ewtOv1RGNGOfOqOSykydw/tvHUJLQX5iBy2X9gGjLe2/rHBoHLM+bzrRAa4MfVM1e4GT890PNZ9v8O9M6cDnI5bxgymUhl+ndkDoSFs0Lhqg3xFdwPub9LHIZOkLN7OBhE437n43mTfv7icQ77z+a8JZbtHNYWuTgwVdwWcwLzvZlsZR3FX/Evx7HzKu/rcn798nl/F5j0nuPpfZPR/2e4SAKSl1Z3E9s3FHPwy+8wWOr3+DNvS2k4hHeMb2S9xw/lnfNHM2IMh1wDaVC4ZDL+sHRk2VZb7gu0+wFUbbNW5/NeO+5dA/ms/t/0Reaz2b2/6LHeeGWX0su7W+fzZv2959Nd9ln1+lMsf8FDsL8Xl1eOOQHTXch1d6bO2B5ZP98oswbpnXOO6svNcT/+fn/prnM/n/z9h7khFNhyt8dWWsUBP1LNud47vU6frdmO79fu4Pt+1owg+PHD+HsaZWcPW0UsycPO/LTUEUGEue8UHB+ILYHjMsdXhjmukxnWqCpzptuP0/FIt4v4FjK+2Wcbd3fc+t4b/GHFQusy99/13o6Lc91+WWev8yfb23whjzNvN5mT5z9Wfj7rx7Rj1lB0I/lco6X3tjL8g21PLVpFy9sfYtMzhExmDGmgtmThnmvycOYPrqi905JFZH+o2m3FzYdx44inafbgy8a944DHQEFwQDS2JphxZbd/PVve1hd7b32NntXFZclosyaOJTZk4Zz8uRhnDxp2OHf2kJEQkk3nRtAypIx3nnsaN557GgAnHNsqWtidfVbHeHw46c2k856AT6qPMmxY8uZPrqCY8dWMGNMBdPHlDMkpRvWiUjPKAj6OTOjalQZVaPKeN/JEwFoSWdZu20fq6v3sO7NfWzYUc+DK6tpatt/Bsq4oSmOqSxjysgypowoZcrIUiaPKGPKyFLKdOxBRPLoN8IAlIpHOXXKcE6dMrxjWS7neGNPMxt21LN+Rz0bdzSwubaBpS+/yZ6mdKfPjypPMmVkKVNGlDJ5ZOeQGFmWCP5KaBHpVxQEg0QkYkwaUcqkEaWc9/YxndbtbU7zt7omtu5uZGtdE1vrvPdnNtfxyOo3yD9MlIhGGD0kybihKcYMSTF2SIqxQ/3XEG/ZmCEpEjE95VRksFAQhMDQkjizJg5l1sShB6xrSWepeauJrXVNVO9u4s19LezY28L2fS2seWMvf3x1By3p3AGfG1WeYMyQFOOGpqisSFFZnqCyIkllRZJR5fvfNQwl0v/pf2nIpeJRpo2uYNroioLrnXPsa87w5r5mtu9tYce+FrbvbWW7P1/zVjOrq/dQ19hGoRPQShNRRpQlGFGWYHhpguGlcYaWxBlamvDeS+IMK4kztNR/L4kzpCROKq4rrkX6ioJAumVmDC31flHPHDvkoNtlsjl2N7VRW9/Krgbvvf31VlMbuxu91+u7GtnT1EZ9a6ZgcLRLxSN+SPiBUZoXGiVxhpV6geFNe9sMScUoT8VIxhQiIodDQSC9IhaNMLoixeiKnl3XkM056lvS7G1Os6fJe9/bnGZPc5p9zWn2NLV1Wle9u4k1/nxzuvv788SjRnnSC4XyZJzyZNSf96bLEjHKkjHKklFKE3nv/nRZMkZpwtuuNBklEY3oALoMagoCKYpoxBhWmmBYaYIpIw/vs62ZLHv9wGgPiz1Naepb0jS2ZWlozdDQkvHe/eldDW1sqWuiviVDY2vmkGGSLxYxShNemJQmY5QlvOAoSUQpSUQpjUcpTURJxaMk41FS8QipmLeufdpbFyEVj/rz/nTe9hFdNS5FoiCQAScZizK6Itrj3kch2ZyjOZ2lqTVDY1uWxtYMTf57Y1uGptas9563rqE1Q1NbhsbWLE1tGXbsS9PclvX247+3ZQ48sN5TiViEVKxLQPjBkcwLjpL8EIlF/PDpHDqJWIR41EhEI8RjEWIRIx6N+Mvz1vnr41EjHokojEJKQSChFI34w0e9fFZTLudozeRoSWdpyWRpSfvTaX86k6Wlreu6/du3Fti+uS3LvpYMtfWtnffjT/em9sCIRy0vNPyg6BIk8WjkgDDpmM/bxlt/4Lqu+0907KdwUHX9vIbreo+CQKQXRSLWMWTUF5zzgqc1LzRaMznS2Rxt2RzpTI501u2fb39lXOd5f5v26bZM53X797V/vqE107Gvzvt3pDPefFs21+1JAUejU9BEIySidtDezwGhFTFiUSMW9aajEW+7WNSIRbx9xPzPRiP523n78rbzto36vamYv2004q2LmHVsF+3Yp7+PvM91bF/E3piCQGQAM7OOYaKh9M/7S2VzrmAw5QfQ4YZWT4Iq//P16QyZXOfQymSdtyzrOmrM5LzpYogYXrBE9gdILLo/2GJR4+q5k/nYOcf0+ncrCEQkUN5fydEBc22Ic45MznUERSbrSOdyZP1l6WzODw5/ffu22RxZ5wVJNuc6QsWbbg8ef7kfOu3b5O8zm7/PnLffTM4Lt6Aec6sgEBHJY2b+cBJ4T9kd/HTDGBGRkFMQiIiEnIJARCTkFAQiIiGnIBARCTkFgYhIyCkIRERCTkEgIhJy5oK6EUhAzKwW2HqEHx8F7OrFcgYCtTkc1OZwOJo2T3HOVRZaMeCC4GiY2Urn3Jxi19GX1OZwUJvDIag2a2hIRCTkFAQiIiEXtiC4u9gFFIHaHA5qczgE0uZQHSMQEZEDha1HICIiXSgIRERCLjRBYGbzzGy9mW0ys0XFrqe3mNk9ZrbTzNbkLRthZn8ws43++/C8dV/0fwbrzew9xan66JjZJDP7s5m9amZrzexmf/mgbbeZpczseTN70W/z1/zlg7bNAGYWNbO/mtmv/flB3V4AM9tiZi+b2WozW+kvC7bdzrlB/8J7zNBrwDFAAngROK7YdfVS294BnAKsyVv2f4FF/vQi4Jv+9HF+25NAlf8ziRa7DUfQ5nHAKf50BbDBb9ugbTdgQLk/HQeeA84YzG322/E54H+AX/vzg7q9flu2AKO6LAu03WHpEcwFNjnnNjvn2oAHgPlFrqlXOOeWA7u7LJ4P/Jc//V/AZXnLH3DOtTrnXgc24f1sBhTn3JvOuRf86XrgVWACg7jdztPgz8b9l2MQt9nMJgIXAz/KWzxo23sIgbY7LEEwAajOm6/xlw1WY5xzb4L3SxMY7S8fdD8HM5sKnIz3F/Kgbrc/TLIa2An8wTk32Nv8XeALQC5v2WBubzsH/N7MVpnZjf6yQNsdlofXW4FlYTxvdlD9HMysHHgY+Ixzbp9ZoeZ5mxZYNuDa7ZzLArPNbBjwiJmd0M3mA7rNZnYJsNM5t8rM3tmTjxRYNmDa28VZzrltZjYa+IOZretm215pd1h6BDXApLz5icC2ItXSF3aY2TgA/32nv3zQ/BzMLI4XAvc5537pLx707QZwzu0BngDmMXjbfBZwqZltwRvKfbeZ/YzB294Ozrlt/vtO4BG8oZ5A2x2WIFgBTDezKjNLAAuAJUWuKUhLgA/50x8CHstbvsDMkmZWBUwHni9CfUfFvD/9fwy86pz7dt6qQdtuM6v0ewKYWQnw98A6BmmbnXNfdM5NdM5Nxfv/+ifn3LUM0va2M7MyM6tonwYuANYQdLuLfYS8D4/EX4R3dslrwJeKXU8vtut+4E0gjffXwUeBkcDjwEb/fUTe9l/yfwbrgQuLXf8RtvlsvO7vS8Bq/3XRYG43cCLwV7/Na4Cv+MsHbZvz2vFO9p81NKjbi3dm44v+a23776qg261bTIiIhFxYhoZEROQgFAQiIiGnIBARCTkFgYhIyCkIRERCTkEgoWVmT/vvU83s6l7e9z8V+i6R/kinj0ro+bcw+Lxz7pLD+EzUebd8ONj6BudceS+UJxI49QgktMys/W6etwPn+Pd//6x/c7dvmdkKM3vJzP7B3/6d/nMQ/gd42V/2qH9zsLXtNwgzs9uBEn9/9+V/l3m+ZWZr/HvOX5W37yfM7CEzW2dm91k3N08S6U1huemcSHcWkdcj8H+h73XOnWZmSeAvZvZ7f9u5wAnOu+UvwEecc7v92z6sMLOHnXOLzOwm59zsAt91OTAbOAkY5X9mub/uZOB4vHvF/AXvfjtP9XZjRbpSj0DkQBcA1/u3fH4O7/L+6f665/NCAODTZvYi8Czezb+m072zgfudc1nn3A7gSeC0vH3XOOdyeLfNmNoLbRE5JPUIRA5kwKecc8s6LfSOJTR2mf974EznXJOZPQGkerDvg2nNm86i/5/SR9QjEIF6vEdetlsGfMK/1TVmNsO/E2RXQ4G3/BCYiffoyHbp9s93sRy4yj8OUYn3qNEBd5dMGVz0F4eId0fPjD/Ecy/wPbxhmRf8A7a17H80YL7fAQvN7CW8Oz8+m7fubuAlM3vBOXdN3vJHgDPx7i7pgC8457b7QSJSFDp9VEQk5DQ0JCIScgoCEZGQUxCIiIScgkBEJOQUBCIiIacgEBEJOQWBiEjI/X+ytNklzSWPrQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(minibatch_net.losses)\n",
    "plt.plot(minibatch_net.val_losses)\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('iteration')\n",
    "plt.legend(['train_loss', 'val_loss'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### softmax 계산 예제"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-1.5666989  -1.21854359 -0.87038828 -0.52223297 -0.17407766  0.17407766\n",
      "  0.52223297  0.87038828  1.21854359  1.5666989 ]\n",
      "[ 2.00904515 -0.77811264  0.70180826  0.54938733 -0.17521064 -0.66711635\n",
      "  0.9088237   0.0159026   0.02923104 -0.45334518]\n",
      "-3.368965282795019\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "x = np.array([1,2,3,4,5,6,7,8,9,10])\n",
    "x = (x - np.mean(x)) / np.std(x)\n",
    "print(x)\n",
    "\n",
    "w = np.random.normal(0,1,10)\n",
    "print(w)\n",
    "\n",
    "z = np.dot(x,w)\n",
    "print(z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2.2  1.39 0.85]\n",
      "[9.0250135  4.01485005 2.33964685]\n",
      "15.379510404354313\n",
      "[0.5868206  0.26105188 0.15212752]\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "temp = np.array([2.20,1.39,0.85])\n",
    "\n",
    "print(temp)\n",
    "exp_temp = np.exp(temp)\n",
    "print(exp_temp)\n",
    "print(np.sum(exp_temp))\n",
    "ret = exp_temp / np.sum(exp_temp)\n",
    "print( ret )\n",
    "print( np.sum(ret) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[9.0250135  4.01485005 2.33964685]\n",
      " [1.         0.2490753  0.11080316]]\n",
      "[15.3795104   1.35987846]\n",
      "(2,)\n",
      "[[0.5868206  0.26105188 0.15212752]\n",
      " [0.73535983 0.18315997 0.08148019]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "temp = np.array([[2.20,1.39,0.85],\n",
    "                 [0.00,-1.39,-2.20]])\n",
    "exp_temp = np.exp(temp)\n",
    "print(exp_temp)\n",
    "temp_sum = np.sum(exp_temp, axis=1)\n",
    "print(temp_sum)\n",
    "print(temp_sum.shape)\n",
    "ret = exp_temp / np.sum(exp_temp, axis=1).reshape(-1, 1)\n",
    "print( ret )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(z):\n",
    "    # 소프트맥스 함수\n",
    "    z = np.clip(z, -100, None)            # 안전한 np.exp() 계산을 위해\n",
    "    exp_z = np.exp(z)\n",
    "    return exp_z / np.sum(exp_z, axis=1).reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.5868206  0.26105188 0.15212752]\n",
      " [0.73535983 0.18315997 0.08148019]]\n",
      "[1. 1.]\n"
     ]
    }
   ],
   "source": [
    "temp = np.array([[2.20,1.39,0.85],\n",
    "                 [0.00,-1.39,-2.20]])\n",
    "a = softmax(temp)\n",
    "print(a)\n",
    "print(np.sum(a, axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}