effb5-512.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "necessary-conservative",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-05-05T08:39:43.295457Z",
     "iopub.status.busy": "2021-05-05T08:39:43.294674Z",
     "iopub.status.idle": "2021-05-05T08:40:06.463671Z",
     "shell.execute_reply": "2021-05-05T08:40:06.462928Z"
    },
    "papermill": {
     "duration": 23.203953,
     "end_time": "2021-05-05T08:40:06.463899",
     "exception": false,
     "start_time": "2021-05-05T08:39:43.259946",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install --quiet ../input/keras-efficientnet-whl/Keras_Applications-1.0.8-py3-none-any.whl\n",
    "!pip install --quiet ../input/keras-efficientnet-whl/efficientnet-1.1.1-py3-none-any.whl\n",
    "\n",
    "import os\n",
    "import math\n",
    "import re\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import tensorflow_addons as tfa\n",
    "import numpy as np\n",
    "import tensorflow.keras.backend as K\n",
    "import efficientnet.keras as efn\n",
    "import efficientnet\n",
    "import itertools\n",
    "import matplotlib\n",
    "import scipy\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "from matplotlib import pyplot as plt\n",
    "from datetime import datetime\n",
    "from functools import partial\n",
    "from kaggle_datasets import KaggleDatasets\n",
    "import pickle\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# import cudf\n",
    "# import cuml\n",
    "# import cupy\n",
    "# from cuml.feature_extraction.text import TfidfVectorizer\n",
    "# from cuml.neighbors import NearestNeighbors\n",
    "\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "SEED = 42\n",
    "\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "tf.random.set_seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "active-subscriber",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:40:06.496740Z",
     "iopub.status.busy": "2021-05-05T08:40:06.496014Z",
     "iopub.status.idle": "2021-05-05T08:41:13.037312Z",
     "shell.execute_reply": "2021-05-05T08:41:13.037845Z"
    },
    "papermill": {
     "duration": 66.561306,
     "end_time": "2021-05-05T08:41:13.038030",
     "exception": false,
     "start_time": "2021-05-05T08:40:06.476724",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device:  grpc://10.0.0.2:8470\n",
      "TPU\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['gs://kds-987282d7bffe5a80b2cc4a3ed7d2235e7ea86e412c97d7c4b82f8bb2/train.tfrec']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "try:\n",
    "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
    "    print('Device: ', tpu.master())\n",
    "    tf.config.experimental_connect_to_cluster(tpu)\n",
    "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
    "    DEVICE = \"TPU\"\n",
    "except:\n",
    "    DEVICE = 'notTPU'\n",
    "    strategy = tf.distribute.get_strategy()\n",
    "print(DEVICE)\n",
    "\n",
    "AUTO = tf.data.experimental.AUTOTUNE\n",
    "GCS_PATH = KaggleDatasets().get_gcs_path(f'shopee-tfrecord-512-all-010521')\n",
    "REPLICAS = strategy.num_replicas_in_sync\n",
    "BATCH_SIZE = 32 * REPLICAS\n",
    "FILENAMES = tf.io.gfile.glob(GCS_PATH + f'/*.tfrec')\n",
    "FILENAMES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "potential-acting",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:41:13.068927Z",
     "iopub.status.busy": "2021-05-05T08:41:13.068276Z",
     "iopub.status.idle": "2021-05-05T08:41:13.071140Z",
     "shell.execute_reply": "2021-05-05T08:41:13.071598Z"
    },
    "papermill": {
     "duration": 0.020919,
     "end_time": "2021-05-05T08:41:13.071803",
     "exception": false,
     "start_time": "2021-05-05T08:41:13.050884",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "IMAGE_SIZE = (512, 512)\n",
    "\n",
    "EPOCHS = 100\n",
    "LR = 0.001\n",
    "\n",
    "CLASSES = 11014"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "static-topic",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:41:13.112586Z",
     "iopub.status.busy": "2021-05-05T08:41:13.111891Z",
     "iopub.status.idle": "2021-05-05T08:41:13.114981Z",
     "shell.execute_reply": "2021-05-05T08:41:13.115424Z"
    },
    "papermill": {
     "duration": 0.030502,
     "end_time": "2021-05-05T08:41:13.115597",
     "exception": false,
     "start_time": "2021-05-05T08:41:13.085095",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def decode_image(image, img_size):\n",
    "    image = tf.image.decode_jpeg(image, channels=3)\n",
    "    image = tf.image.resize(image, img_size)\n",
    "    image = tf.cast(image, tf.float32) / 255.0\n",
    "    return tf.reshape(image, [*img_size, 3])\n",
    "\n",
    "# parse tfrecord files\n",
    "def _parse_image(proto, train):\n",
    "    features = {\n",
    "        \"image\": tf.io.FixedLenFeature([], tf.string), # tf.string means bytestring\n",
    "        \"title_vec\": tf.io.FixedLenFeature([100], tf.float32),  # shape [] means single element\n",
    "        \"phash_vec\": tf.io.FixedLenFeature([25], tf.float32),\n",
    "        \"label\": tf.io.FixedLenFeature([], tf.int64),  \n",
    "    }\n",
    "    return tf.io.parse_single_example(proto, features)\n",
    "\n",
    "# get image and label_group\n",
    "def get_image_and_label(proto, train, img_size):\n",
    "    sample = _parse_image(proto, train=train)\n",
    "    sample[\"image\"] = decode_image(sample[\"image\"], img_size)\n",
    "    image = sample[\"image\"]\n",
    "    title_vec = sample[\"title_vec\"]\n",
    "    phash_vec = sample[\"phash_vec\"]\n",
    "    label = tf.cast(sample['label'], tf.int32)\n",
    "#     label = tf.one_hot(tf.cast(sample['label'], tf.int32), len(CLASSES))\n",
    "    return (image, title_vec, label), label\n",
    "\n",
    "def get_training_dataset(tfr, batchsize, img_size = (512,512)):\n",
    "    return tfr.map(partial(get_image_and_label, train=True, img_size=img_size)).repeat().map(\n",
    "        lambda x,y: image_augmentation(x,y)).shuffle(1000).batch(batchsize).prefetch(AUTO)\n",
    "\n",
    "def get_validation_dataset(tfr, batchsize, img_size=(512, 512), train=True):\n",
    "    return tfr.map(partial(get_image_and_label, train=train, img_size=img_size)).batch(batchsize).prefetch(AUTO)\n",
    "\n",
    "def get_tfrecord_size(tfrecord):\n",
    "    return sum(1 for _ in tfrecord)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "narrow-springer",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:41:13.149186Z",
     "iopub.status.busy": "2021-05-05T08:41:13.148528Z",
     "iopub.status.idle": "2021-05-05T08:41:13.152170Z",
     "shell.execute_reply": "2021-05-05T08:41:13.151633Z"
    },
    "papermill": {
     "duration": 0.023329,
     "end_time": "2021-05-05T08:41:13.152308",
     "exception": false,
     "start_time": "2021-05-05T08:41:13.128979",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Augmentation function & tf.image functions (flip, brightness, contrast, saturation, hue)\n",
    "# for train\n",
    "# @tf.function\n",
    "def image_augmentation(iw, label):\n",
    "    image, title_vec, label = iw\n",
    "\n",
    "    image = tf.image.random_flip_left_right(image)\n",
    "    image = tf.image.random_flip_up_down(image)\n",
    "    image = tf.image.random_hue(image, 0.01)\n",
    "    image = tf.image.random_saturation(image, 0.70, 1.30)\n",
    "    image = tf.image.random_contrast(image, 0.80, 1.20)\n",
    "    image = tf.image.random_brightness(image, 0.10)\n",
    "\n",
    "    return ((image, title_vec, label), label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "happy-bullet",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:41:13.185351Z",
     "iopub.status.busy": "2021-05-05T08:41:13.184698Z",
     "iopub.status.idle": "2021-05-05T08:41:13.187877Z",
     "shell.execute_reply": "2021-05-05T08:41:13.188343Z"
    },
    "papermill": {
     "duration": 0.022999,
     "end_time": "2021-05-05T08:41:13.188527",
     "exception": false,
     "start_time": "2021-05-05T08:41:13.165528",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Model, Sequential\n",
    "import keras.backend as K\n",
    "from keras.optimizers import SGD\n",
    "from tensorflow.keras.layers import (\n",
    "    Input, \n",
    "    Flatten, \n",
    "    Dense, \n",
    "    Dropout, \n",
    "    AveragePooling2D, \n",
    "    GlobalAveragePooling2D, \n",
    "    SpatialDropout2D, \n",
    "    BatchNormalization, \n",
    "    Activation, \n",
    "    Concatenate,\n",
    "    Embedding,\n",
    "    GlobalAveragePooling1D,\n",
    "    Lambda\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fallen-referral",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:41:13.228707Z",
     "iopub.status.busy": "2021-05-05T08:41:13.227971Z",
     "iopub.status.idle": "2021-05-05T08:41:13.231738Z",
     "shell.execute_reply": "2021-05-05T08:41:13.231094Z"
    },
    "papermill": {
     "duration": 0.029891,
     "end_time": "2021-05-05T08:41:13.231894",
     "exception": false,
     "start_time": "2021-05-05T08:41:13.202003",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def plt_lr(epoch_count):\n",
    "    if epoch_count > 50:\n",
    "        epoch_count = 50\n",
    "    \n",
    "    rng = [i for i in range(epoch_count)]\n",
    "\n",
    "    plt.figure()\n",
    "    y = [lrfn(x) for x in rng]\n",
    "    plt.title(f'Learning rate schedule: {y[0]} to {y[epoch_count-1]}')\n",
    "    plt.plot(rng, y)\n",
    "\n",
    "def plt_acc(h):\n",
    "    plt.figure()\n",
    "    plt.plot(h.history[\"sparse_categorical_accuracy\"])\n",
    "    if 'val_sparse_categorical_accuracy' in h.history:\n",
    "        plt.plot(h.history[\"val_sparse_categorical_accuracy\"]) \n",
    "        plt.legend([\"training\",\"validation\"])       \n",
    "    else:\n",
    "        plt.legend([\"training\"])\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.title(\"Sparse Categorical Accuracy\")\n",
    "    plt.show()\n",
    "    \n",
    "def plt_f1_score(h):\n",
    "    plt.figure()\n",
    "    plt.plot(h.history[\"f1_score\"])\n",
    "    if 'f1_score' in h.history:\n",
    "        plt.plot(h.history[\"f1_score\"]) \n",
    "        plt.legend([\"training\",\"validation\"])       \n",
    "    else:\n",
    "        plt.legend([\"training\"])\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.title(\"F1 Score\")\n",
    "    plt.show()\n",
    "\n",
    "def plt_loss(h):\n",
    "    plt.figure()\n",
    "    plt.plot(h.history[\"loss\"])\n",
    "    if 'val_loss' in h.history:\n",
    "        plt.plot(h.history[\"val_loss\"]) \n",
    "        plt.legend([\"training\",\"validation\"])       \n",
    "    else:\n",
    "        plt.legend([\"training\"])\n",
    "    plt.legend([\"training\",\"validation\"])\n",
    "    plt.xlabel(\"epoch\")\n",
    "    plt.title(\"Loss\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "animated-trading",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:41:13.266724Z",
     "iopub.status.busy": "2021-05-05T08:41:13.266054Z",
     "iopub.status.idle": "2021-05-05T08:41:13.269289Z",
     "shell.execute_reply": "2021-05-05T08:41:13.268706Z"
    },
    "papermill": {
     "duration": 0.023967,
     "end_time": "2021-05-05T08:41:13.269428",
     "exception": false,
     "start_time": "2021-05-05T08:41:13.245461",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "es_val_f1 = tf.keras.callbacks.EarlyStopping(\n",
    "    monitor='f1_score', min_delta=0.001, patience=10, verbose=1, mode='max',\n",
    "    baseline=None, restore_best_weights=True\n",
    ")\n",
    "\n",
    "es_val_f1_stage2 = tf.keras.callbacks.EarlyStopping(\n",
    "    monitor='f1_score', min_delta=0.001, patience=15, verbose=1, mode='max',\n",
    "    baseline=None, restore_best_weights=True\n",
    ")\n",
    "\n",
    "cb_checkpoint = tf.keras.callbacks.ModelCheckpoint(\n",
    "    \"backup.h5\",\n",
    "    monitor=\"f1_score\", verbose=1, mode='max', save_best_only=True)\n",
    "cb_checkpoint._supports_tf_logs = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ultimate-border",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:41:13.303696Z",
     "iopub.status.busy": "2021-05-05T08:41:13.302907Z",
     "iopub.status.idle": "2021-05-05T08:41:13.314054Z",
     "shell.execute_reply": "2021-05-05T08:41:13.313301Z"
    },
    "papermill": {
     "duration": 0.030866,
     "end_time": "2021-05-05T08:41:13.314214",
     "exception": false,
     "start_time": "2021-05-05T08:41:13.283348",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# swish activation\n",
    "from keras.backend import sigmoid\n",
    "def swish(x, beta = 1):\n",
    "    return (x * sigmoid(beta * x))\n",
    "\n",
    "from keras.utils.generic_utils import get_custom_objects\n",
    "from keras.layers import Activation\n",
    "\n",
    "get_custom_objects().update({'swish': Activation(swish)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "meaning-walker",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:41:13.347829Z",
     "iopub.status.busy": "2021-05-05T08:41:13.347178Z",
     "iopub.status.idle": "2021-05-05T08:41:13.350649Z",
     "shell.execute_reply": "2021-05-05T08:41:13.350091Z"
    },
    "papermill": {
     "duration": 0.022712,
     "end_time": "2021-05-05T08:41:13.350827",
     "exception": false,
     "start_time": "2021-05-05T08:41:13.328115",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, multiply, Reshape\n",
    "from keras.layers import Bidirectional, GlobalMaxPool1D,Bidirectional, Conv1D, GlobalMaxPooling1D, Conv2D\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "otherwise-mountain",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:41:13.396895Z",
     "iopub.status.busy": "2021-05-05T08:41:13.395850Z",
     "iopub.status.idle": "2021-05-05T08:41:13.399427Z",
     "shell.execute_reply": "2021-05-05T08:41:13.398880Z"
    },
    "papermill": {
     "duration": 0.034844,
     "end_time": "2021-05-05T08:41:13.399559",
     "exception": false,
     "start_time": "2021-05-05T08:41:13.364715",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ArcMarginProduct(keras.layers.Layer):\n",
    "    '''\n",
    "    Implements large margin arc distance.\n",
    "\n",
    "    Reference:\n",
    "        https://arxiv.org/pdf/1801.07698.pdf\n",
    "        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n",
    "            blob/master/src/modeling/metric_learning.py\n",
    "    '''\n",
    "    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n",
    "                 ls_eps=0.0, **kwargs):\n",
    "\n",
    "        super(ArcMarginProduct, self).__init__(**kwargs)\n",
    "\n",
    "        self.n_classes = n_classes\n",
    "        self.s = s\n",
    "        self.m = m\n",
    "        self.ls_eps = ls_eps\n",
    "        self.easy_margin = easy_margin\n",
    "        self.cos_m = tf.math.cos(m)\n",
    "        self.sin_m = tf.math.sin(m)\n",
    "        self.th = tf.math.cos(math.pi - m)\n",
    "        self.mm = tf.math.sin(math.pi - m) * m\n",
    "\n",
    "    def get_config(self):\n",
    "\n",
    "        config = super().get_config().copy()\n",
    "        config.update({\n",
    "            'n_classes': self.n_classes,\n",
    "            's': self.s,\n",
    "            'm': self.m,\n",
    "            'ls_eps': self.ls_eps,\n",
    "            'easy_margin': self.easy_margin,\n",
    "        })\n",
    "        return config\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        super(ArcMarginProduct, self).build(input_shape[0])\n",
    "\n",
    "        self.W = self.add_weight(\n",
    "            name='W',\n",
    "            shape=(int(input_shape[0][-1]), self.n_classes),\n",
    "            initializer='glorot_uniform',\n",
    "            dtype='float32',\n",
    "            trainable=True,\n",
    "            regularizer=None)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        X, y = inputs\n",
    "        y = tf.cast(y, dtype=tf.int32)\n",
    "        cosine = tf.matmul(\n",
    "            tf.math.l2_normalize(X, axis=1),\n",
    "            tf.math.l2_normalize(self.W, axis=0)\n",
    "        )\n",
    "        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n",
    "        phi = cosine * self.cos_m - sine * self.sin_m\n",
    "        if self.easy_margin:\n",
    "            phi = tf.where(cosine > 0, phi, cosine)\n",
    "        else:\n",
    "            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n",
    "        one_hot = tf.cast(\n",
    "            tf.one_hot(y, depth=self.n_classes),\n",
    "            dtype=cosine.dtype\n",
    "        )\n",
    "        if self.ls_eps > 0:\n",
    "            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n",
    "\n",
    "        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n",
    "        output *= self.s\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "super-surrey",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:41:13.446668Z",
     "iopub.status.busy": "2021-05-05T08:41:13.441810Z",
     "iopub.status.idle": "2021-05-05T08:41:13.449818Z",
     "shell.execute_reply": "2021-05-05T08:41:13.449248Z"
    },
    "papermill": {
     "duration": 0.036355,
     "end_time": "2021-05-05T08:41:13.449961",
     "exception": false,
     "start_time": "2021-05-05T08:41:13.413606",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from keras.layers import LeakyReLU\n",
    "\n",
    "def effb5(weights='noisy-student'):\n",
    "    img_inp = Input(shape=(*IMAGE_SIZE, 3))\n",
    "    effb5 = efn.EfficientNetB5(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    \n",
    "    pt_depth = effb5.layers[-1].get_output_shape_at(0)[-1]\n",
    "    pt_features = effb5(img_inp)\n",
    "    bn_features = BatchNormalization()(pt_features)\n",
    "    \n",
    "    attn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(Dropout(0.5)(bn_features))\n",
    "    attn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(8, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(1, kernel_size = (1,1), padding = 'valid', activation = 'sigmoid')(attn_layer)\n",
    "    \n",
    "    up_c2_w = np.ones((1, 1, 1, pt_depth))\n",
    "    up_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', activation = 'linear', \n",
    "                   use_bias = False, weights = [up_c2_w])\n",
    "    up_c2.trainable = False\n",
    "    attn_layer = up_c2(attn_layer)\n",
    "    \n",
    "    mask_features = multiply([attn_layer, bn_features])\n",
    "    gap_features = GlobalAveragePooling2D()(mask_features)\n",
    "    gap_mask = GlobalAveragePooling2D()(attn_layer)\n",
    "    # to account for missing values from the attention model\n",
    "    gap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\n",
    "    gap_dr = Dropout(0.25)(gap)\n",
    "    img_embed = Dense(2400, activation = 'swish', name='img-embed')(gap_dr)\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([img_embed, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[img_inp, model_title.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "# model = effb5()\n",
    "# model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "fatal-martial",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:41:13.486402Z",
     "iopub.status.busy": "2021-05-05T08:41:13.485509Z",
     "iopub.status.idle": "2021-05-05T08:41:13.488874Z",
     "shell.execute_reply": "2021-05-05T08:41:13.488305Z"
    },
    "papermill": {
     "duration": 0.024883,
     "end_time": "2021-05-05T08:41:13.489017",
     "exception": false,
     "start_time": "2021-05-05T08:41:13.464134",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_lr_callback():\n",
    "    lr_start   = 0.000001\n",
    "    lr_max     = 0.000005 * BATCH_SIZE\n",
    "    lr_min     = 0.000001\n",
    "    lr_ramp_ep = 5\n",
    "    lr_sus_ep  = 0\n",
    "    lr_decay   = 0.8\n",
    "   \n",
    "    def lrfn(epoch):\n",
    "        if epoch < lr_ramp_ep:\n",
    "            lr = (lr_max - lr_start) / lr_ramp_ep * epoch + lr_start   \n",
    "        elif epoch < lr_ramp_ep + lr_sus_ep:\n",
    "            lr = lr_max    \n",
    "        else:\n",
    "            lr = (lr_max - lr_min) * lr_decay**(epoch - lr_ramp_ep - lr_sus_ep) + lr_min    \n",
    "        return lr\n",
    "\n",
    "    lr_callback = tf.keras.callbacks.LearningRateScheduler(lrfn, verbose = True)\n",
    "    return lr_callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "completed-equation",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:41:13.520604Z",
     "iopub.status.busy": "2021-05-05T08:41:13.519980Z",
     "iopub.status.idle": "2021-05-05T08:42:09.942710Z",
     "shell.execute_reply": "2021-05-05T08:42:09.941558Z"
    },
    "papermill": {
     "duration": 56.439714,
     "end_time": "2021-05-05T08:42:09.942934",
     "exception": false,
     "start_time": "2021-05-05T08:41:13.503220",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "\n",
    "# for randomness\n",
    "ignore_order = tf.data.Options()\n",
    "ignore_order.experimental_deterministic = False\n",
    "\n",
    "train_records = tf.data.TFRecordDataset(FILENAMES, num_parallel_reads=AUTO)\n",
    "train_records = train_records.with_options(ignore_order)\n",
    "train_data = get_training_dataset(train_records, BATCH_SIZE, IMAGE_SIZE)\n",
    "train_data = train_data.map(lambda x, y: (x, y))    \n",
    "train_size = get_tfrecord_size(train_records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "skilled-norwegian",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-05T08:42:09.976510Z",
     "iopub.status.busy": "2021-05-05T08:42:09.975825Z",
     "iopub.status.idle": "2021-05-05T14:06:59.550299Z",
     "shell.execute_reply": "2021-05-05T14:06:59.549012Z"
    },
    "papermill": {
     "duration": 19489.591602,
     "end_time": "2021-05-05T14:06:59.550468",
     "exception": false,
     "start_time": "2021-05-05T08:42:09.958866",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://github.com/qubvel/efficientnet/releases/download/v0.0.1/efficientnet-b5_noisy-student_notop.h5\n",
      "115261440/115255328 [==============================] - 2s 0us/step\n",
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 512, 512, 3) 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "efficientnet-b5 (Functional)    (None, 16, 16, 2048) 28513520    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization (BatchNorma (None, 16, 16, 2048) 8192        efficientnet-b5[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 16, 16, 2048) 0           batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "title-input (InputLayer)        [(None, 100)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv2d (Conv2D)                 (None, 16, 16, 64)   131136      dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "title-embed (Embedding)         (None, 100, 150)     3750000     title-input[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_1 (Conv2D)               (None, 16, 16, 16)   1040        conv2d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 100, 150)     0           title-embed[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_2 (Conv2D)               (None, 16, 16, 8)    136         conv2d_1[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d (Conv1D)                 (None, 98, 300)      135300      dropout_2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_3 (Conv2D)               (None, 16, 16, 1)    9           conv2d_2[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_max_pooling1d (GlobalMax (None, 300)          0           conv1d[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_4 (Conv2D)               (None, 16, 16, 2048) 2048        conv2d_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "title-dense1 (Dense)            (None, 720)          216720      global_max_pooling1d[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "multiply (Multiply)             (None, 16, 16, 2048) 0           conv2d_4[0][0]                   \n",
      "                                                                 batch_normalization[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "title-act1 (Activation)         (None, 720)          0           title-dense1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d (Globa (None, 2048)         0           multiply[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_1 (Glo (None, 2048)         0           conv2d_4[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 720)          0           title-act1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "RescaleGAP (Lambda)             (None, 2048)         0           global_average_pooling2d[0][0]   \n",
      "                                                                 global_average_pooling2d_1[0][0] \n",
      "__________________________________________________________________________________________________\n",
      "title-dense2 (Dense)            (None, 650)          468650      dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 2048)         0           RescaleGAP[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "title-bn2 (BatchNormalization)  (None, 650)          2600        title-dense2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "img-embed (Dense)               (None, 2400)         4917600     dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "title-act2 (Activation)         (None, 650)          0           title-bn2[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 3050)         0           img-embed[0][0]                  \n",
      "                                                                 title-act2[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "arc-input (InputLayer)          [(None,)]            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "head/arc_margin (ArcMarginProdu (None, 11014)        33592700    concatenate[0][0]                \n",
      "                                                                 arc-input[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "output (Dense)                  (None, 11014)        121319210   head/arc_margin[0][0]            \n",
      "==================================================================================================\n",
      "Total params: 193,058,861\n",
      "Trainable params: 192,878,681\n",
      "Non-trainable params: 180,180\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/40\n",
      "\n",
      "Epoch 00001: LearningRateScheduler reducing learning rate to 1e-06.\n",
      "133/133 [==============================] - 663s 3s/step - loss: 9.4733 - sparse_categorical_accuracy: 9.5567e-05\n",
      "Epoch 2/40\n",
      "\n",
      "Epoch 00002: LearningRateScheduler reducing learning rate to 0.00025680000000000006.\n",
      "133/133 [==============================] - 470s 4s/step - loss: 9.3449 - sparse_categorical_accuracy: 0.0043\n",
      "Epoch 3/40\n",
      "\n",
      "Epoch 00003: LearningRateScheduler reducing learning rate to 0.0005126000000000001.\n",
      "133/133 [==============================] - 474s 4s/step - loss: 8.2278 - sparse_categorical_accuracy: 0.0670\n",
      "Epoch 4/40\n",
      "\n",
      "Epoch 00004: LearningRateScheduler reducing learning rate to 0.0007684000000000001.\n",
      "133/133 [==============================] - 473s 4s/step - loss: 5.9939 - sparse_categorical_accuracy: 0.2078\n",
      "Epoch 5/40\n",
      "\n",
      "Epoch 00005: LearningRateScheduler reducing learning rate to 0.0010242.\n",
      "133/133 [==============================] - 479s 4s/step - loss: 3.5099 - sparse_categorical_accuracy: 0.4688\n",
      "Epoch 6/40\n",
      "\n",
      "Epoch 00006: LearningRateScheduler reducing learning rate to 0.00128.\n",
      "133/133 [==============================] - 480s 4s/step - loss: 1.4636 - sparse_categorical_accuracy: 0.7845\n",
      "Epoch 7/40\n",
      "\n",
      "Epoch 00007: LearningRateScheduler reducing learning rate to 0.0010242.\n",
      "133/133 [==============================] - 480s 4s/step - loss: 0.4343 - sparse_categorical_accuracy: 0.9274\n",
      "Epoch 8/40\n",
      "\n",
      "Epoch 00008: LearningRateScheduler reducing learning rate to 0.0008195600000000003.\n",
      "133/133 [==============================] - 480s 4s/step - loss: 0.1928 - sparse_categorical_accuracy: 0.9633\n",
      "Epoch 9/40\n",
      "\n",
      "Epoch 00009: LearningRateScheduler reducing learning rate to 0.0006558480000000003.\n",
      "133/133 [==============================] - 480s 4s/step - loss: 0.1175 - sparse_categorical_accuracy: 0.9792\n",
      "Epoch 10/40\n",
      "\n",
      "Epoch 00010: LearningRateScheduler reducing learning rate to 0.0005248784000000002.\n",
      "133/133 [==============================] - 484s 4s/step - loss: 0.0774 - sparse_categorical_accuracy: 0.9870\n",
      "Epoch 11/40\n",
      "\n",
      "Epoch 00011: LearningRateScheduler reducing learning rate to 0.0004201027200000002.\n",
      "133/133 [==============================] - 484s 4s/step - loss: 0.0443 - sparse_categorical_accuracy: 0.9917\n",
      "Epoch 12/40\n",
      "\n",
      "Epoch 00012: LearningRateScheduler reducing learning rate to 0.0003362821760000002.\n",
      "133/133 [==============================] - 486s 4s/step - loss: 0.0399 - sparse_categorical_accuracy: 0.9936\n",
      "Epoch 13/40\n",
      "\n",
      "Epoch 00013: LearningRateScheduler reducing learning rate to 0.00026922574080000017.\n",
      "133/133 [==============================] - 483s 4s/step - loss: 0.0303 - sparse_categorical_accuracy: 0.9951\n",
      "Epoch 14/40\n",
      "\n",
      "Epoch 00014: LearningRateScheduler reducing learning rate to 0.00021558059264000014.\n",
      "133/133 [==============================] - 483s 4s/step - loss: 0.0224 - sparse_categorical_accuracy: 0.9970\n",
      "Epoch 15/40\n",
      "\n",
      "Epoch 00015: LearningRateScheduler reducing learning rate to 0.0001726644741120001.\n",
      "133/133 [==============================] - 484s 4s/step - loss: 0.0161 - sparse_categorical_accuracy: 0.9981\n",
      "Epoch 16/40\n",
      "\n",
      "Epoch 00016: LearningRateScheduler reducing learning rate to 0.0001383315792896001.\n",
      "133/133 [==============================] - 483s 4s/step - loss: 0.0113 - sparse_categorical_accuracy: 0.9984\n",
      "Epoch 17/40\n",
      "\n",
      "Epoch 00017: LearningRateScheduler reducing learning rate to 0.00011086526343168008.\n",
      "133/133 [==============================] - 484s 4s/step - loss: 0.0105 - sparse_categorical_accuracy: 0.9986\n",
      "Epoch 18/40\n",
      "\n",
      "Epoch 00018: LearningRateScheduler reducing learning rate to 8.889221074534406e-05.\n",
      "133/133 [==============================] - 479s 4s/step - loss: 0.0082 - sparse_categorical_accuracy: 0.9989\n",
      "Epoch 19/40\n",
      "\n",
      "Epoch 00019: LearningRateScheduler reducing learning rate to 7.131376859627525e-05.\n",
      "133/133 [==============================] - 483s 4s/step - loss: 0.0063 - sparse_categorical_accuracy: 0.9990\n",
      "Epoch 20/40\n",
      "\n",
      "Epoch 00020: LearningRateScheduler reducing learning rate to 5.725101487702021e-05.\n",
      "133/133 [==============================] - 488s 4s/step - loss: 0.0068 - sparse_categorical_accuracy: 0.9991\n",
      "Epoch 21/40\n",
      "\n",
      "Epoch 00021: LearningRateScheduler reducing learning rate to 4.600081190161617e-05.\n",
      "133/133 [==============================] - 482s 4s/step - loss: 0.0045 - sparse_categorical_accuracy: 0.9993\n",
      "Epoch 22/40\n",
      "\n",
      "Epoch 00022: LearningRateScheduler reducing learning rate to 3.7000649521292936e-05.\n",
      "133/133 [==============================] - 485s 4s/step - loss: 0.0046 - sparse_categorical_accuracy: 0.9994\n",
      "Epoch 23/40\n",
      "\n",
      "Epoch 00023: LearningRateScheduler reducing learning rate to 2.9800519617034356e-05.\n",
      "133/133 [==============================] - 480s 4s/step - loss: 0.0036 - sparse_categorical_accuracy: 0.9996\n",
      "Epoch 24/40\n",
      "\n",
      "Epoch 00024: LearningRateScheduler reducing learning rate to 2.4040415693627487e-05.\n",
      "133/133 [==============================] - 484s 4s/step - loss: 0.0033 - sparse_categorical_accuracy: 0.9996\n",
      "Epoch 25/40\n",
      "\n",
      "Epoch 00025: LearningRateScheduler reducing learning rate to 1.9432332554901988e-05.\n",
      "133/133 [==============================] - 482s 4s/step - loss: 0.0032 - sparse_categorical_accuracy: 0.9996\n",
      "Epoch 26/40\n",
      "\n",
      "Epoch 00026: LearningRateScheduler reducing learning rate to 1.574586604392159e-05.\n",
      "133/133 [==============================] - 484s 4s/step - loss: 0.0032 - sparse_categorical_accuracy: 0.9997\n",
      "Epoch 27/40\n",
      "\n",
      "Epoch 00027: LearningRateScheduler reducing learning rate to 1.2796692835137275e-05.\n",
      "133/133 [==============================] - 478s 4s/step - loss: 0.0029 - sparse_categorical_accuracy: 0.9996\n",
      "Epoch 28/40\n",
      "\n",
      "Epoch 00028: LearningRateScheduler reducing learning rate to 1.043735426810982e-05.\n",
      "133/133 [==============================] - 481s 4s/step - loss: 0.0032 - sparse_categorical_accuracy: 0.9996\n",
      "Epoch 29/40\n",
      "\n",
      "Epoch 00029: LearningRateScheduler reducing learning rate to 8.549883414487856e-06.\n",
      "133/133 [==============================] - 481s 4s/step - loss: 0.0026 - sparse_categorical_accuracy: 0.9997\n",
      "Epoch 30/40\n",
      "\n",
      "Epoch 00030: LearningRateScheduler reducing learning rate to 7.0399067315902855e-06.\n",
      "133/133 [==============================] - 476s 4s/step - loss: 0.0030 - sparse_categorical_accuracy: 0.9996\n",
      "Epoch 31/40\n",
      "\n",
      "Epoch 00031: LearningRateScheduler reducing learning rate to 5.831925385272228e-06.\n",
      "133/133 [==============================] - 481s 4s/step - loss: 0.0027 - sparse_categorical_accuracy: 0.9997\n",
      "Epoch 32/40\n",
      "\n",
      "Epoch 00032: LearningRateScheduler reducing learning rate to 4.865540308217783e-06.\n",
      "133/133 [==============================] - 486s 4s/step - loss: 0.0026 - sparse_categorical_accuracy: 0.9998\n",
      "Epoch 33/40\n",
      "\n",
      "Epoch 00033: LearningRateScheduler reducing learning rate to 4.092432246574227e-06.\n",
      "133/133 [==============================] - 481s 4s/step - loss: 0.0026 - sparse_categorical_accuracy: 0.9998\n",
      "Epoch 34/40\n",
      "\n",
      "Epoch 00034: LearningRateScheduler reducing learning rate to 3.473945797259381e-06.\n",
      "133/133 [==============================] - 482s 4s/step - loss: 0.0026 - sparse_categorical_accuracy: 0.9997\n",
      "Epoch 35/40\n",
      "\n",
      "Epoch 00035: LearningRateScheduler reducing learning rate to 2.979156637807505e-06.\n",
      "133/133 [==============================] - 484s 4s/step - loss: 0.0028 - sparse_categorical_accuracy: 0.9995\n",
      "Epoch 36/40\n",
      "\n",
      "Epoch 00036: LearningRateScheduler reducing learning rate to 2.5833253102460042e-06.\n",
      "133/133 [==============================] - 479s 4s/step - loss: 0.0026 - sparse_categorical_accuracy: 0.9997\n",
      "Epoch 37/40\n",
      "\n",
      "Epoch 00037: LearningRateScheduler reducing learning rate to 2.2666602481968035e-06.\n",
      "133/133 [==============================] - 479s 4s/step - loss: 0.0030 - sparse_categorical_accuracy: 0.9995\n",
      "Epoch 38/40\n",
      "\n",
      "Epoch 00038: LearningRateScheduler reducing learning rate to 2.013328198557443e-06.\n",
      "133/133 [==============================] - 484s 4s/step - loss: 0.0024 - sparse_categorical_accuracy: 0.9998\n",
      "Epoch 39/40\n",
      "\n",
      "Epoch 00039: LearningRateScheduler reducing learning rate to 1.8106625588459543e-06.\n",
      "133/133 [==============================] - 475s 4s/step - loss: 0.0025 - sparse_categorical_accuracy: 0.9997\n",
      "Epoch 40/40\n",
      "\n",
      "Epoch 00040: LearningRateScheduler reducing learning rate to 1.6485300470767634e-06.\n",
      "133/133 [==============================] - 484s 4s/step - loss: 0.0025 - sparse_categorical_accuracy: 0.9998\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEWCAYAAAB2X2wCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmo0lEQVR4nO3deXhc9X3v8fdXu2TLm7yAVxkwYEODA8ZA4BI3gcSmFGhD2ZoU8pDSy4VbciEpJs0lCbe9DSVJCw0pISkNISQEEkgc4lwDYStbsAwOwTYYg0e2vGuxkCXL1vK9f5wj+2jQMpJGOjOjz+t55vGcZc75zk/WR7/5nTPnmLsjIiLZLy/uAkREJD0U6CIiOUKBLiKSIxToIiI5QoEuIpIjFOgiIjlCgS4yQGb2GzO7cojbuMrMXkhXTSKgQM96ZnaWmb1kZo1mVm9mL5rZqXHX1R8zW2xmK81sb1j3q2b22RRf+6yZfW64a+yNuy9z9/uHez9mNtbM9pnZb4Z7X5IbFOhZzMzGAY8D/wZMAmYAXwMODMO+CtK4rTOAp4HngGOACuBaYFm69jEcLDCSvzOfIvhZnmtmR4zgftP685YR5O56ZOkDWATs7WP5VcCLwLeBRuAt4OOR5Z8FNgBNwHvA30SWLQFqgJuBncADwGSCPyB7gXrgv4C8cP3pwM+BPcBm4G/7qOsF4O4+lk8M97MHaAifzwyX/SPQAbQC+4Bvh/OPB54M63obuCSyvQrgV8D7wGrgH4AXIss/Es5vDP/9SGTZs+E+XwT2E/wBehb4XGSdv46043rg5HD+cuDdyPw/S/rZvNBbG4TrPB3u+zXgC0nLzgJeCn8WW4GrwvmlwDeB6vD9vBDOWwLUJG0jAZwTPv8q8DPgR2E7fQ5YDLwc7mMHwf+josjrT4i0+S7gS8ARQAtQEVnv5PBnWRj370yuP2IvQI8h/PBgHFAH3E/Qu52YtPwqoB34X0AhcGn4Sz4pXP4nwNGAAR8NfxG7wmhJ+NrbgeIwFP4JuCfcViHw38LX5gFrgFuBIuAogj8Qn+yh5jKCQP7jPt5XBUHvtAwoBx4BfhFZnhyoY8JQ+yxQAHwYqAUWhMsfCh9lwIJw3RfCZZMI/mh8Jnzt5eF0RWRfW8LwKgjf96H9A38BbANODdviGGBOZNn0sH0uBZqBIyM/m14DHZgDdIb13gS8kbSsKay1MGyvheGyu8P6ZgD5BH+sikkt0NuAi8J6S4FTgNPD911J8Efr8+H65QQhfxNQEk6fFi5bCVwb2c+/AP8W9+/LaHjEXoAeQ/wBwnzgBwS96XZgBTAtXHYVsB2wyPqvAp/pZVu/AG4Iny8BDgIlkeW3Ab8Ejkl63WnAlqR5twD/2cM+ZgAOHD+A97gQaIhMHwrUcPpS4L+SXvNd4CthqLUBx0WWHeqhEwT5q0mvfZnDPd5ngduSlh/aP7Cqq81SeB9rgQsjP5u+Av3LwNpIm3UAH4607WM9vCaP4FPEST0sW0L/gf58P/V/vmu/BH9MXu9lvUuBF8Pn+QSf8BaP1O/EaH5oDD3LufsGd7/K3WcCJxL0CP81sso2D3+zQtXhOpjZMjN7JTwouRc4j2BYpcsed2+NTN8BbAKeMLP3zGx5OH8OMD08wLk33NaXgGk9lNxA0PM8srf3ZGZlZvZdM6s2s/eB54EJZpbfy0vmAKcl7f8vCT7+TyHoYW6NrB99Pj1sk6hqghDtaf1kswiGVXp6H39lZmsjNZ1I9/bty18BDwK4+zaC4w1dZ9b0ts/JBL3lHutJQbf3aWbHmtnjZrYz/Dn8Xw7X3+v7Jvijv8DM5gLnAo3u/uoga5IBUKDnEHd/i6C3fmJk9gwzs8j0bGC7mRUTjHl/g6BHP4Hgo3J03W6X4nT3Jne/yd2PAi4AbjSzjxMEwWZ3nxB5lLv7eT3U2ELQA/5UH2/lJuA4go/w44Czw/ldtSVfInQr8FzS/se6+7UEY7ftwMzI+rMiz7cT/EGImk0wjNJjO/Sw76OTZ5rZHOB7wPUEwzcTgDfp3r49MrOPAPOAW8Iw3UnwKeiK8GBlj/skGGZq7WVZM8GQU9c+8gn+2EUlv89/JzjuMi/8OXwpUv9WgqG1Dwg7AQ8Dnyb4BPRAz+9U0k2BnsXM7Hgzu8nMZobTswg+Cr8SWW0q8LdmVmhmf0EwRLOSYKy7mDDwzGwZ8Il+9ne+mR0T/oFoJBgG6CQYxmkys5vNrNTM8s3sxD5On/w74Coz+6KZVYTbPsnMHgqXlxMMHew1s0kEQydRu+geJo8Dx5rZZ8L3WWhmp5rZfHfvAB4Fvhr2/I8n6P12WRm+9gozKzCzSwnGrR/vqy0ivg98wcxOCc+COSYM8zEEAbknfH+fpfsf2r5cSXCwcQHBcNPC8LWlBMdKHgTOMbNLwporzGyhu3cC9wHfMrPp4c/hjPCP90agxMz+xMwKCYZ0ivupo5zgAOm+sN2ujSx7HDjSzD5vZsVmVm5mp0WW/5BgWOkCFOgjRoGe3ZoIem6/M7NmgiB/k6CH2+V3BL29WoIzJi529zp3bwL+lqAn1QBcQTD+3pd5wFMEZ5e8DHzH3Z8JQ/N8guDZHO7r+8D4njbi7i8BHwsf75lZPXAvQbhCMGRUGm7nFeD/JW3iTuBiM2sws7vC9/IJ4DKCHvdODh/MhaCXPJ7DZ+v8hPDUTnevC2u/ieAA898B57t7bT9t0fVeHiFo1x8T/Dx+QXDQeT3B2SYvE/wB+iOCM2X6ZGYlwCUEBxF3Rh6bw9qvdPctBMNjNxGcYbIWOCncxBeAPxCcrVMftkOeuzcC/4Pg57KNoMde0085XyD4f9FE8Gnjp5H33UQwnPKnBO36DvDHkeUvEvyxf83dk4e0ZJhY9+FVySVmdhXBwbuz4q4lk5jZ7cAR7j6kb3tK38zsaeDH7v79uGsZLdRDl5wXDk19KBwSWQxcDTwWd125LBxuO5lIr16Gn74NJqNBOcEwy3SC4Y9vEpyJIcPAzO4nOJ/9hnBoRkaIhlxERHKEhlxERHJEbEMukydP9srKyrh2LyKSldasWVPr7snfIQBiDPTKykqqqqri2r2ISFYys15PA9WQi4hIjlCgi4jkCAW6iEiOUKCLiOQIBbqISI7oN9DN7D4z221mb/ay3MzsLjPbZGZvmNnJ6S9TRET6k0oP/QfA0j6WLyO4Ct884BqCayiLiMgI6/c8dHd/3swq+1jlQuCH4V1xXjGzCWZ2pLvvSFeRkj06O52DHZ20dzpt7Z20dXQG0x1OpztO120PCZ8TzA//7eh0Otzp7Iw+p9trIbwTg4MTbuvQ9j64j+j2Cac7O6PLutfjOJ3hxKF50f1yeH9021fwum776jx8aY2u24xE7zdiBtbHPS+63t+haT88nXzRjuhWLGmT3baRNN/DOdH3FH2BmR2qM8/CmsN5h2rpoQ173OdgLzUSbbOkWUZXfYfnm1m3mqLvNbmE6Hai0z22TZoulfLx+dM4adaEtGwrKh1fLJpB91tX1YTzPhDoZnYNQS+e2bNnp2HXMhBtHZ3sb+ug9WAHLQc72N/WcWh6f1sHzQc7aD7QHj46aD7Yzr5D0+3sb+vgQFsnB9o7OdDeQWtb8O+B9k5a2zpo6whCWHJDNLDTvd2ByLTLTQ20/p5MHVeSsYGeMne/l+BGBixatCjDfkzZbf/BDt6r3ceOva3sfL+VXeFj5/sH2NUYzGvc3zagbZYU5jG2uICyogLGFBdQVpRPcUEe5SUFlBQGz4sL8ikpzKMofF6QbxTm51GUn0dhvlEQPi/IN/K6enoW9IXMCOZFnufnGXl5Rn7X80P/dv0iWaRH1X1bh3pqkedd+8wLX5vXbd4H6+nqhWLde35d60BSjy58nrzdYN7h1yf3Eruep6r33qiF2zq8seQeeLS9SHrd4Xb84PzD24j2vsNPIZGervXSVl1tOxwOfWKK1AXdPyEk1xR9r8ntlvypx927vQ8YvveSTukI9G10v0fjTLrfj1HSqL2jk0RdCxt3NfHWziY27mzi7V1NJOqauwWEGUwZW8wR40uYNamMU+dOZMrYEsYU51NSmE9ZUT6lhfmUFOVTVphPaVEwb0xxGN6F+RTk6ySo7DF8YdM1vDLc+xmIQwE9xLo+uJ2hbS9u6Qj0FcD14f0gTyO4w7fGz9NoR+N+Hv/9Dn79hx2s3/E+B9s7gaBXWFkxhuOmlXPBSdOZN20sMyaUcsT4EqaMLVYgi4wy/Qa6mf0EWAJMNrMaghv2FgK4+z0E94E8D9gEtACfHa5iR5OG5oOsfHMHK9Zu59VEPe7wRzPGc+UZczjuiHEcN62cedPGUlKYH3epIpIhUjnL5fJ+ljtwXdoqGsVaDrbz5PpdrFi7nec27qG90zlqyhg+//FjuWDhdOZOHhN3iSKSwXQLugyxprqe6x58nZ3vt3Lk+BKuPmsuf3rSdE6YPi4rDsaISPwU6DFzd+57McE/rdzA9Aml/Phzp3H6URXk5SnERWRgFOgx2negnZt/9ga//sMOzpk/jW9echLjSwvjLktEspQCPSYbdzXx33+0hkRtM8uXHc/fnH2UhlZEZEgU6DH4xevbuOXRPzCmuIAHP3c6ZxxdEXdJIpIDFOgj6EB7B//w+AYeeKWaxZWT+PYVH2bquJK4yxKRHKFAH0FdYX7N2UfxxU8eR6G++CMiaaRAHyHtHZ08/sZ2LjhpOl86b37c5YhIDlIXcYS8mqinoaWNZSceEXcpIpKjFOgjZNWbOykuyOOjx02JuxQRyVEK9BHg7jyxfhdnHzuFsiKNconI8FCgj4A3ahrZ0djKJ0/QcIuIDB8F+ghYtW4n+XnGOfOnxl2KiOQwBfoIWLVuJ6fNncSEsqK4SxGRHKZAH2abdjfx7p5mlursFhEZZgr0YbZq3S4APrFAgS4iw0uBPsxWrdvJSbMmcMR4fcVfRIaXAn0Ybd+7nzdqGvnkCdPiLkVERgEF+jB6Yt1OAJ2uKCIjQoE+jFat28W8qWM5esrYuEsRkVFAgT5M6psP8mqiXr1zERkxCvRh8tSGXXR0ugJdREaMAn2YPLFuJzMmlHLijHFxlyIio4QCfRg0H2jn+XdqOXfBNN0nVERGjAJ9GDy3cQ8H2zs13CIiI0qBPgxWrdvJpDFFnFo5Me5SRGQUUaCn2cH2Tp7esJtz5k+lQPcMFZERpMRJs5feraXpQLuGW0RkxCnQ02zVul2MKcrnzGMmx12KiIwyCvQ06uh0nly/iyXHTaWkMD/uckRklFGgp9HrWxqo3XeAT+ra5yISg5QC3cyWmtnbZrbJzJb3sHy2mT1jZq+b2Rtmdl76S818v9tcD8BHj50ScyUiMhr1G+hmlg/cDSwDFgCXm9mCpNW+DDzs7h8GLgO+k+5Cs0Gitpmp5cWMLy2MuxQRGYVS6aEvBja5+3vufhB4CLgwaR0Hur7jPh7Ynr4Ss0d1XQuVFWPiLkNERqlUAn0GsDUyXRPOi/oq8GkzqwFWAv+zpw2Z2TVmVmVmVXv27BlEuZktUdfMnIqyuMsQkVEqXQdFLwd+4O4zgfOAB8zsA9t293vdfZG7L5oyJbfGmVsOtrO76YACXURik0qgbwNmRaZnhvOirgYeBnD3l4ESYFSdiL2lvgWAORpyEZGYpBLoq4F5ZjbXzIoIDnquSFpnC/BxADObTxDouTem0odEbRDoGkMXkbj0G+ju3g5cD6wCNhCczbLOzG4zswvC1W4C/trMfg/8BLjK3X24is5E1XXNAMzWkIuIxKQglZXcfSXBwc7ovFsjz9cDZ6a3tOxSXd/CxLJCnbIoIrHRN0XTpLquWePnIhIrBXqaJGpbqNRwi4jESIGeBgfaO9jeuJ/Z6qGLSIwU6GlQ07Afd9RDF5FYKdDToOsMF42hi0icFOhpcPgcdPXQRSQ+CvQ0qK5rZmxxAZPGFMVdioiMYgr0NKiub2FORRlmFncpIjKKKdDTQJfNFZFMoEAfovaOTraGPXQRkTgp0IdoR2Mr7Z2uQBeR2CnQhyihUxZFJEMo0IcoUafL5opIZlCgD1F1bTPFBXlMLS+OuxQRGeUU6EPUdcpiXp5OWRSReCnQh0iXzRWRTKFAH4LOTg/PQdcZLiISPwX6EOxqauVAe6cumysiGUGBPgTVdbool4hkDgX6EHRdNlenLIpIJlCgD0GiroXCfOPI8SVxlyIiokAfiuq6ZmZOLKMgX80oIvFTEg1BdZ0uyiUimUOBPkjursvmikhGUaAPUl3zQfYdaGf2JPXQRSQzKNAH6dAZLpMV6CKSGRTog9R1Drq+9i8imUKBPkiJuhbyDGZOLI27FBERQIE+aNV1zRw5vpTigvy4SxERARTog5aoa9H4uYhklJQC3cyWmtnbZrbJzJb3ss4lZrbezNaZ2Y/TW2bm2aLL5opIhinobwUzywfuBs4FaoDVZrbC3ddH1pkH3AKc6e4NZjZ1uArOBI0tbTS0tOmiXCKSUVLpoS8GNrn7e+5+EHgIuDBpnb8G7nb3BgB3353eMjNLdX1wyuLsSeqhi0jmSCXQZwBbI9M14byoY4FjzexFM3vFzJb2tCEzu8bMqsysas+ePYOrOAMcujG0xtBFJIOk66BoATAPWAJcDnzPzCYkr+Tu97r7IndfNGXKlDTteuRtqevqoSvQRSRzpBLo24BZkemZ4byoGmCFu7e5+2ZgI0HA56REXQtTy4spK+r3EISIyIhJJdBXA/PMbK6ZFQGXASuS1vkFQe8cM5tMMATzXvrKzCzVdc26KJeIZJx+A93d24HrgVXABuBhd19nZreZ2QXhaquAOjNbDzwDfNHd64ar6LgldNlcEclAKY0ZuPtKYGXSvFsjzx24MXzktJaD7expOkDlZPXQRSSz6JuiA9R1US4dEBWRTKNAHyDdGFpEMpUCfYC6zkGfrTF0EckwCvQBqq5rYdKYIsaXFsZdiohINwr0Aaqua9b4uYhkJAX6AAU3hlagi0jmUaAPwIH2DrY37tdlc0UkIynQB2Br/X7c0ZeKRCQjKdAHIFEbnLKoHrqIZCIF+gAkwnPQ5+pboiKSgRToA7C5tplxJQVMLNMpiyKSeRToA1Bd18LcyWMws7hLERH5AAX6AGyubdZFuUQkYynQU9TaFpyyqGu4iEimUqCnaGt9C+46ICoimUuBnqLNh05Z1DnoIpKZFOgp0imLIpLpFOgp2lzbwoSyQiaUFcVdiohIjxToKUrU6sbQIpLZFOgpStQ1a7hFRDKaAj0F+w92sKOxVT10EcloCvQUbKkPbjtXOVlnuIhI5lKgp6DrlEUNuYhIJlOgp6DrlEV97V9EMpkCPQWJ2mYqxhQxrkRXWRSRzKVAT8Hm2mZ9Q1REMp4CPQWJOl1lUUQynwK9Hy0H29n1/gHm6pRFEclwCvR+JGq7TllUoItIZlOg90MX5RKRbKFA74dOWRSRbJFSoJvZUjN728w2mdnyPtb7lJm5mS1KX4nxStQ2M3lsMWOLC+IuRUSkT/0GupnlA3cDy4AFwOVmtqCH9cqBG4DfpbvIOCVqW5irr/yLSBZIpYe+GNjk7u+5+0HgIeDCHtb7P8DtQGsa64vd5jpdNldEskMqgT4D2BqZrgnnHWJmJwOz3P3XfW3IzK4xsyozq9qzZ8+Aix1p+w60s6fpgMbPRSQrDPmgqJnlAd8CbupvXXe/190XufuiKVOmDHXXwy4RXpRLPXQRyQapBPo2YFZkemY4r0s5cCLwrJklgNOBFblwYPTwGS4aQxeRzJdKoK8G5pnZXDMrAi4DVnQtdPdGd5/s7pXuXgm8Alzg7lXDUvEIUg9dRLJJv4Hu7u3A9cAqYAPwsLuvM7PbzOyC4S4wTptrW5haXswYnbIoIlkgpaRy95XAyqR5t/ay7pKhl5UZdFEuEckm+qZoH6rrmnVRLhHJGgr0XjS1tlG776B66CKSNRTovei6yqK+JSoi2UKB3ovNuiiXiGQZBXovuk5ZnDNJgS4i2UGB3otEbTNHjCuhtCg/7lJERFKiQO/F5rpmfUNURLKKAr0Xidpm3aVIRLKKAr0HjS1tNLS06Sv/IpJVFOg90G3nRCQbKdB7oBtDi0g2UqD3YHNtM2Ywe5IOiopI9lCg9yBR28z08aWUFOqURRHJHgr0Hmyua9EpiyKSdRToPUjU6sbQIpJ9FOhJGpoP0rhfpyyKSPZRoCfRRblEJFsp0JN0XZRLl80VkWyjQE+SqGshz2CWTlkUkSyjQE+SqG1m+oRSigt0yqKIZBcFepJEnS7KJSLZSYEe4e5s1imLIpKlFOgROxpbaWpt5+gpCnQRyT4K9IjViXoATpkzKeZKREQGToEeUZVoYExRPvOPLI+7FBGRAVOgR6xO1HPynIkU5KtZRCT7KLlCjfvbeHtXE4s03CIiWUqBHnqtugF3OHXuxLhLEREZFAV6aHWinoI8Y+GsCXGXIiIyKAr00OpEPSfMGE9ZUUHcpYiIDEpKgW5mS83sbTPbZGbLe1h+o5mtN7M3zOy3ZjYn/aUOn9a2Dn6/tZHFlRpuEZHs1W+gm1k+cDewDFgAXG5mC5JWex1Y5O4fAn4G/HO6Cx1Ob25r5GBHJ4sqdUBURLJXKj30xcAmd3/P3Q8CDwEXRldw92fcvSWcfAWYmd4yh9er4ReKFs1RD11EslcqgT4D2BqZrgnn9eZq4Dc9LTCza8ysysyq9uzZk3qVw6wq0cDRU8ZQMbY47lJERAYtrQdFzezTwCLgjp6Wu/u97r7I3RdNmTIlnbsetM5OpypRz6kabhGRLJfKKR3bgFmR6ZnhvG7M7Bzg74GPuvuB9JQ3/N7ZvY/3W9s1fi4iWS+VHvpqYJ6ZzTWzIuAyYEV0BTP7MPBd4AJ3353+ModP1/j5YgW6iGS5fgPd3duB64FVwAbgYXdfZ2a3mdkF4Wp3AGOBR8xsrZmt6GVzGacqUc/U8mJmTSqNuxQRkSFJ6Vs07r4SWJk079bI83PSXNeIqUo0cGrlJMws7lJERIZkVH9TdNve/Wzbu59T9YUiEckBozrQq7rOP9f4uYjkgFEd6KsT9YwtLmD+kePiLkVEZMhGd6BvbuDkORPJz9P4uYhkv1Eb6I0twQ0tTtXX/UUkR4zaQF+zRePnIpJbRm2gv7q5gcJ83dBCRHLHqA30qkQ9J84YT2lRftyliIikxagM9Na2Dt6oadQFuUQkp4zKQH+jJryhhQ6IikgOGZWBvlpfKBKRHDQqA70qUc8xU8cyaUxR3KWIiKTNqAv0zk6nqrpB128RkZwz6gL97V1NNLW264CoiOScURfoXRfkUqCLSK4ZdYG+OtHAtHHFzJyoG1qISG4ZVYHeuL+N5zbu4fSjKnRDCxHJOaMq0L/73Ls07m/jmrOPirsUEZG0GzWBvrOxlfte3MxFC6dzwvTxcZcjIpJ2oybQ7/ztRjo6nZs+cVzcpYiIDItREeibdu/jp6u38unT5zBrUlnc5YiIDIuCuAsYCXeseouyogKu/+Nj4i5FZNRra2ujpqaG1tbWuEvJaCUlJcycOZPCwsKUX5Pzgb6muoFV63Zx47nHUjG2OO5yREa9mpoaysvLqays1NlmvXB36urqqKmpYe7cuSm/LqeHXNyd23/zFpPHFnP1Wak3iogMn9bWVioqdOpwX8yMioqKAX+KyelAf+bt3byaqOeGc+YxpjjnP4yIZA2Fef8G00Y5G+gdnc7tv3mbyooyLjt1VtzliIgMu5wN9Mde38bbu5r44iePpzA/Z9+miAzQ3r17+c53vjPg15133nns3bu3z3VuvfVWnnrqqUFWNnQ5mXStbR1864m3OWnmeM77oyPiLkdEMkhvgd7e3t7n61auXMmECRP6XOe2227jnHPOGUp5Q5KTA8sPvFzN9sZWvnHJSRqrE8lgX/vVOtZvfz+t21wwfRxf+dMTel2+fPly3n33XRYuXEhhYSElJSVMnDiRt956i40bN3LRRRexdetWWltbueGGG7jmmmsAqKyspKqqin379rFs2TLOOussXnrpJWbMmMEvf/lLSktLueqqqzj//PO5+OKLqays5Morr+RXv/oVbW1tPPLIIxx//PHs2bOHK664gu3bt3PGGWfw5JNPsmbNGiZPnjzk955zPfTG/W18+5lNfPTYKXzk6KE3kIjklq9//escffTRrF27ljvuuIPXXnuNO++8k40bNwJw3333sWbNGqqqqrjrrruoq6v7wDbeeecdrrvuOtatW8eECRP4+c9/3uO+Jk+ezGuvvca1117LN77xDQC+9rWv8bGPfYx169Zx8cUXs2XLlrS9t5zpoXd2Oi++W8u9z7/H+61t3Lz0+LhLEpF+9NWTHimLFy/udq73XXfdxWOPPQbA1q1beeedd6ioqOj2mrlz57Jw4UIATjnlFBKJRI/b/vM///ND6zz66KMAvPDCC4e2v3TpUiZOTN/d01IKdDNbCtwJ5APfd/evJy0vBn4InALUAZe6eyJtVfZha30Lj6yp4edrati2dz/jSwu5ZdnxLJg+biR2LyJZbsyYMYeeP/vsszz11FO8/PLLlJWVsWTJkh7PBS8uPvwlxfz8fPbv39/jtrvWy8/P73eMPh36DXQzywfuBs4FaoDVZrbC3ddHVrsaaHD3Y8zsMuB24NLhKBiCg56r1u3k4aqtvLipDjM465jJLF92POcumEZJYf5w7VpEslx5eTlNTU09LmtsbGTixImUlZXx1ltv8corr6R9/2eeeSYPP/wwN998M0888QQNDQ1p23YqPfTFwCZ3fw/AzB4CLgSigX4h8NXw+c+Ab5uZubunrdLQT1dv4R9+vYGm1nZmTizlxnOP5VOnzGTGBN2BSET6V1FRwZlnnsmJJ55IaWkp06ZNO7Rs6dKl3HPPPcyfP5/jjjuO008/Pe37/8pXvsLll1/OAw88wBlnnMERRxxBeXl5WrZt/WWumV0MLHX3z4XTnwFOc/frI+u8Ga5TE06/G65Tm7Sta4BrAGbPnn1KdXX1gAt+fuMeHn2thksWzeL0oyrIy9NZLCLZZMOGDcyfPz/uMmJz4MAB8vPzKSgo4OWXX+baa69l7dq1Pa7bU1uZ2Rp3X9TT+iN6UNTd7wXuBVi0aNGgeu9nHzuFs4+dkta6RERGypYtW7jkkkvo7OykqKiI733ve2nbdiqBvg2Ifnd+Zjivp3VqzKwAGE9wcFRERCLmzZvH66+/PizbTuU89NXAPDOba2ZFwGXAiqR1VgBXhs8vBp4ejvFzEckNiof+DaaN+g10d28HrgdWARuAh919nZndZmYXhKv9B1BhZpuAG4HlA65EREaFkpIS6urqFOp96LoeeklJyYBe1+9B0eGyaNEir6qqimXfIhIf3bEoNb3dsShjDoqKiBQWFg7oLjySupy7louIyGilQBcRyREKdBGRHBHbQVEz2wMM/KuigclAbb9rxUO1DY5qGxzVNjjZXNscd+/x25WxBfpQmFlVb0d546baBke1DY5qG5xcrU1DLiIiOUKBLiKSI7I10O+Nu4A+qLbBUW2Do9oGJydry8oxdBER+aBs7aGLiEgSBbqISI7IukA3s6Vm9raZbTKzjLqqo5klzOwPZrbWzGK98piZ3Wdmu8O7SXXNm2RmT5rZO+G/6bvd+NBr+6qZbQvbbq2ZnRdTbbPM7BkzW29m68zshnB+7G3XR22xt52ZlZjZq2b2+7C2r4Xz55rZ78Lf15+Gl+DOlNp+YGabI+22cKRri9SYb2avm9nj4fTg2s3ds+YB5APvAkcBRcDvgQVx1xWpLwFMjruOsJazgZOBNyPz/hlYHj5fDtyeQbV9FfhCBrTbkcDJ4fNyYCOwIBParo/aYm87wICx4fNC4HfA6cDDwGXh/HuAazOoth8AF8f9fy6s60bgx8Dj4fSg2i3beuiHbljt7geBrhtWSxJ3fx6oT5p9IXB/+Px+4KKRrKlLL7VlBHff4e6vhc+bCO4BMIMMaLs+aoudB/aFk4Xhw4GPEdw4HuJrt95qywhmNhP4E+D74bQxyHbLtkCfAWyNTNeQIf+hQw48YWZrwhtiZ5pp7r4jfL4TmNbXyjG43szeCIdkYhkOijKzSuDDBD26jGq7pNogA9ouHDZYC+wGniT4NL3Xg5vkQIy/r8m1uXtXu/1j2G7/YmbFcdQG/Cvwd0BnOF3BINst2wI9053l7icDy4DrzOzsuAvqjQef5TKmlwL8O3A0sBDYAXwzzmLMbCzwc+Dz7v5+dFncbddDbRnRdu7e4e4LCe47vBg4Po46epJcm5mdCNxCUOOpwCTg5pGuy8zOB3a7+5p0bC/bAj2VG1bHxt23hf/uBh4j+E+dSXaZ2ZEA4b+7Y67nEHffFf7SdQLfI8a2M7NCgsB80N0fDWdnRNv1VFsmtV1Yz17gGeAMYEJ443jIgN/XSG1LwyEsd/cDwH8ST7udCVxgZgmCIeSPAXcyyHbLtkBP5YbVsTCzMWZW3vUc+ATwZt+vGnHRm3lfCfwyxlq66QrL0J8RU9uF45f/AWxw929FFsXedr3VlgltZ2ZTzGxC+LwUOJdgjP8ZghvHQ3zt1lNtb0X+QBvBGPWIt5u73+LuM929kiDPnnb3v2Sw7Rb30d1BHA0+j+Do/rvA38ddT6SuowjOuvk9sC7u2oCfEHz8biMYg7uaYGzut8A7wFPApAyq7QHgD8AbBOF5ZEy1nUUwnPIGsDZ8nJcJbddHbbG3HfAh4PWwhjeBW8P5RwGvApuAR4DiDKrt6bDd3gR+RHgmTFwPYAmHz3IZVLvpq/8iIjki24ZcRESkFwp0EZEcoUAXEckRCnQRkRyhQBcRyREKdJFBMLMlXVfGE8kUCnQRkRyhQJecZmafDq+FvdbMvhtepGlfeDGmdWb2WzObEq670MxeCS/W9FjXRa7M7Bgzeyq8nvZrZnZ0uPmxZvYzM3vLzB4Mv3EoEhsFuuQsM5sPXAqc6cGFmTqAvwTGAFXufgLwHPCV8CU/BG529w8RfIOwa/6DwN3ufhLwEYJvuUJwtcPPE1yT/CiC63KIxKag/1VEstbHgVOA1WHnuZTgolqdwE/DdX4EPGpm44EJ7v5cOP9+4JHw+jwz3P0xAHdvBQi396q714TTa4FK4IVhf1civVCgSy4z4H53v6XbTLP/nbTeYK9/cSDyvAP9PknMNOQiuey3wMVmNhUO3Rd0DsH/+64r2V0BvODujUCDmf23cP5ngOc8uDNQjZldFG6j2MzKRvJNiKRKPQrJWe6+3sy+THAXqTyCqzteBzQT3OTgywRDMJeGL7kSuCcM7PeAz4bzPwN818xuC7fxFyP4NkRSpqstyqhjZvvcfWzcdYikm4ZcRERyhHroIiI5Qj10EZEcoUAXEckRCnQRkRyhQBcRyREKdBGRHPH/ASo4Cy5Jy1QSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEWCAYAAABPON1ZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Z1A+gAAAACXBIWXMAAAsTAAALEwEAmpwYAAAdMUlEQVR4nO3de3BcZ5nn8e/TrZt1sWTr4jgWsWxLCb4lJrETEztMJlzKk7CBBTaTGkLBLJBdCnZgh2VwaiAm1OwWs8MOl6nJDM6FgcmFIQksLJtijYEADk5Ajo2jxAZfYjuyY0uyLVuyLclSP/tHn3ZkWZZacrfO6e7fp6pLp0+fPnr6Lemno/e85z3m7oiISHTFwi5ARETGpqAWEYk4BbWISMQpqEVEIk5BLSIScQpqEZGIU1CLiEScglpympntM7O3hV2HSDYpqEVEIk5BLXnHzErN7Ktmdih4fNXMSoPX6szsR2bWbWbHzOxXZhYLXvusmR00sx4z+72ZvTXcTyKSVBR2ASJZ8NfASmAZ4MAPgM8Bnwc+DbQD9cG2KwE3s6uATwAr3P2QmTUB8aktW2R0OqKWfPR+4Ivu3uHuncB9wAeC184Cs4G57n7W3X/lyQlvhoBSYJGZFbv7PnffE0r1IiMoqCUfXQ7sH/Z8f7AO4O+A3cAGM9trZmsB3H038CngC0CHmX3HzC5HJAIU1JKPDgFzhz2/IliHu/e4+6fdfT5wO/CXqb5od3/M3VcH73Xgb6e2bJHRKaglHxSbWVnqATwOfM7M6s2sDrgXeATAzN5pZs1mZsAJkl0eCTO7ysxuCU469gFngEQ4H0fkfApqyQdPkwzW1KMMaAW2Ay8CLwB/E2zbAmwEeoHNwP3u/nOS/dNfArqAw0ADcM/UfQSRizPdOEBEJNp0RC0iEnEKahGRiFNQi4hEnIJaRCTisnIJeV1dnTc1NWVj1yIieWnLli1d7l4/2mtZCeqmpiZaW1uzsWsRkbxkZvsv9pq6PkREIk5BLSIScQpqEZGI03zUIpIRZ8+epb29nb6+vrBLibSysjIaGxspLi5O+z0KahHJiPb2dqqqqmhqaiI555WM5O4cPXqU9vZ25s2bl/b71PUhIhnR19dHbW2tQnoMZkZtbe2E/+tQUItIxiikxzeZNopUUP/DT3fRdvBE2GWIiERKZIK6+/QAj/3mAO+5/9f86+Z9aPpVEZmo7u5u7r///gm/79Zbb6W7u3vMbe699142btw4ycouTWSCuqa8hP/7FzdxY3Mtn//BS3zi8a309J0NuywRySEXC+rBwcEx3/f0009TU1Mz5jZf/OIXedvb3nYp5U1aZIIaYGZFCQ9/cAWfXfNGftx2mHf+wyZ1hYhI2tauXcuePXtYtmwZK1as4KabbuL2229n0aJFALz73e/muuuuY/Hixaxfv/7c+5qamujq6mLfvn0sXLiQj370oyxevJh3vOMdnDlzBoAPfehDPPnkk+e2X7duHddeey1Lly5l586dAHR2dvL2t7+dxYsX85GPfIS5c+fS1dV1yZ8rcsPzYjHjYzcvYEXTDD7x2Fbec/+v+fw7F3LXyrk6USGSI+77Py/x8qGTGd3nosuns+7fLR5zmy996Uu0tbWxbds2nnnmGW677Tba2trODYV7+OGHmTlzJmfOnGHFihW8973vpba29rx97Nq1i8cff5wHHniAO+64g6eeeoq77rrrgu9VV1fHCy+8wP3338+Xv/xlHnzwQe677z5uueUW7rnnHn784x/z0EMPZeSzR+qIerjlTTN5+pPqChGRybv++uvPG6/89a9/nWuuuYaVK1fy6quvsmvXrgveM2/ePJYtWwbAddddx759+0bd93ve854Lttm0aRN33nknAGvWrGHGjBkZ+RyRO6IeLtUV8o1f7uXLG35P28ETfOfulcyunhZ2aSIyhvGOfKdKRUXFueVnnnmGjRs3snnzZsrLy7n55ptHHc9cWlp6bjkej5/r+rjYdvF4fNw+8EsV2SPqlFRXyGMfuYH9R0/z1Jb2sEsSkYiqqqqip6dn1NdOnDjBjBkzKC8vZ+fOnTz33HMZ//6rVq3iu9/9LgAbNmzg+PHjGdlv5IM65Yb5tSycPZ1ndx8NuxQRiaja2lpWrVrFkiVL+MxnPnPea2vWrGFwcJCFCxeydu1aVq5cmfHvv27dOjZs2MCSJUt44oknuOyyy6iqqrrk/Vo2xisvX77cs3HjgL/50ct8e/N+frfuHUwriWd8/yIyeTt27GDhwoVhlxGq/v5+4vE4RUVFbN68mY997GNs27btgu1Gaysz2+Luy0fbb6T7qEda1VLHg5teoXX/MW5qGfWONSIioTlw4AB33HEHiUSCkpISHnjggYzsN6eC+vqmmRTFjGd3H1VQi0jktLS0sHXr1ozvN2f6qAEqSou49ooZPLv70geQi0jmaeqH8U2mjXIqqAFubK6l7dAJuk8PhF2KiAxTVlbG0aNHFdZjSM1HXVZWNqH35VTXB8Dq5jq+unEXm/cc5U+Wzg67HBEJNDY20t7eTmdnZ9ilRFrqDi8TkXNBfc0baqgoifPsni4FtUiEFBcXT+iuJZK+nOv6KI7HuGF+rcZTi0jByLmgBrhxQS2vdJ3iYPfol3aKiOSTnAzq1S11ABr9ISIFISeD+qpZVdRVlvBrBbWIFICcDGoz48YFdTy7R0OBRCT/5WRQA6xqrqWzp59dHb1hlyIiklU5HNTqpxaRwpCzQd04o5y5teUKahHJezkb1JA8qn5u7zEGhxJhlyIikjW5HdQL6ujtH+R37bpTuYjkr5wO6jcvqMUMDdMTkbyW00E9s6KERbOns0lBLSJ5LKeDGpKz6W090M3pgezeBVhEJCxpBbWZ/Vcze8nM2szscTOb2GSqWXRjcx0DQwl+uy8zd/sVEYmacYPazOYAfwEsd/clQBy4M9uFpWtF0wxK4jH1U4tI3kq366MImGZmRUA5cCh7JU1MeUkRb7qihmf3KKhFJD+NG9TufhD4MnAAeA044e4bRm5nZnebWauZtU71HR5WN9fx0qGTHD+l23OJSP5Jp+tjBvAuYB5wOVBhZneN3M7d17v7cndfXl8/tXcIv7G5DnfYvFc3ExCR/JNO18fbgFfcvdPdzwLfA27MblkTc01jNZWlRRqmJyJ5KZ2gPgCsNLNyMzPgrcCO7JY1MUXxGCvnz9QJRRHJS+n0UT8PPAm8ALwYvGd9luuasBsX1LHv6GkO6fZcIpJn0roLubuvA9ZluZZLcnVjNQC/P9zD5TXTQq5GRCRzcv7KxJTmhkoAdutGAiKSZ/ImqGvKS6irLGVXR0/YpYiIZFTeBDVAc0OFjqhFJO/kVVC3NFSxq6NXN7wVkbySV0Hd3FBJT98gnT39YZciIpIxeRXULcEJRd2ZXETySV4FdWrkx64jOqEoIvkjr4K6vqqU6WVF7O7UEbWI5I+8Cmozo2VWFbuOKKhFJH/kVVADNNdXskdH1CKSR/IuqFtmVdLVO6C5qUUkb+RdUC9IXUquo2oRyRN5F9Tnhuipn1pE8kTeBfXl1dOYVhzXpeQikjfyLqhjMaO5oVKTM4lI3si7oIbkhS97dEQtInkib4P60Ik+evsHwy5FROSS5W1QAzqqFpG8kJdBrcmZRCSf5GVQXzGznJJ4TCcURSQv5GVQF8VjzKurUNeHiOSFvAxqgOZZler6EJG8kL9BXV/Jq8dO03d2KOxSREQuSd4GdcusShIOeztPhV2KiMglydugbtbkTCKSJ/I2qOfVVRAz2K3bcolIjsvboC4tijO3tkJH1CKS8/I2qCHZ/aHpTkUk1+V9UO87eoqzQ4mwSxERmbS8DuqWhkrODjn7j54OuxQRkUnL66A+N/JDl5KLSA7L66BeUJ8KavVTi0juyuugrigtYk7NNF1KLiI5La2gNrMaM3vSzHaa2Q4ze3O2C8uU5oZKHVGLSE5L94j6a8CP3f2NwDXAjuyVlFktDZXs6ewlkfCwSxERmZRxg9rMqoG3AA8BuPuAu3dnua6MaW6opO9sgoPdZ8IuRURkUtI5op4HdALfNLOtZvagmVWM3MjM7jazVjNr7ezszHihk9UyK3W3F438EJHclE5QFwHXAv/k7m8CTgFrR27k7uvdfbm7L6+vr89wmZPXXF8FaOSHiOSudIK6HWh39+eD50+SDO6cUF1eTH1VqS4lF5GcNW5Qu/th4FUzuypY9Vbg5axWlWHN9ZWanElEcla6oz7+C/ComW0HlgH/I2sVZUHLrEp2H+nFXSM/RCT3FKWzkbtvA5Znt5TsaW6opKd/kCMn+7msuizsckREJiSvr0xMeX3OD3V/iEjuKaig1hA9EclFBRHU9ZWlVE8r1hG1iOSkgghqM6OloVKTM4lITiqIoAZNziQiuauggvrYqQGOnxoIuxQRkQkpmKCeX5+cnmRvl46qRSS3FExQp+72sqfjVMiViIhMTMEEdeOMckriMfboUnIRyTEFE9TxmDGvroI9nTqiFpHcUjBBDcl+6r06ohaRHFNQQb2gvpL9x04zMJgIuxQRkbQVVlA3VDCUcA4cU/eHiOSOwgrq1MgP9VOLSA4pqKCeV5ccS62RHyKSSwoqqKvKipk1vVRjqUUkpxRUUEOy+0NH1CKSSwo2qHVbLhHJFQUY1BX09A3S1avJmUQkNxRcUM8/N/JD3R8ikhsKLqgXNCioRSS3FFxQz55exrTiuEZ+iEjOKLigjsUsOeeH5qUWkRxRcEENyX5qdX2ISK4oyKBeUF9B+/Ez9J0dCrsUEZFxFWhQV+IOr3Spn1pEoq9ggxo08kNEckNBBnVqcqa9mkVPRHJAQQb1tJI4c2qm6YhaRHJCQQY1JC98UVCLSC4o3KCur2BPxykSCU3OJCLRVrBBPb++kjNnhzh8si/sUkRExpR2UJtZ3My2mtmPslnQVFlQrxOKIpIbJnJE/UlgR7YKmWrNGqInIjkiraA2s0bgNuDB7JYzdeqrSqkqLVJQi0jkpXtE/VXgr4DExTYws7vNrNXMWjs7OzNRW1aZJSdnUlCLSNSNG9Rm9k6gw923jLWdu6939+Xuvry+vj5jBWbTgvpK9VGLSOSlc0S9CrjdzPYB3wFuMbNHslrVFFnQUMlrJ/ro7R8MuxQRkYsaN6jd/R53b3T3JuBO4GfuflfWK5sCqZEfr+ioWkQirGDHUYPunygiuaFoIhu7+zPAM1mpJARza8uJmYJaRKKtoI+oS4viXDGzXCcURSTSCjqoITnyQ0fUIhJlBR/UyRvdnmJIkzOJSEQVfFAvqK9kYDDBweNnwi5FRGRUCuqGYORHl7o/RCSaFNSpIXodCmoRiaaCD+qZFSXMKC9mj0Z+iEhEFXxQQ/LCF438EJGoUlCTvJR8r4JaRCJKQU2yn7qrd4ATp8+GXYqIyAUU1Aw7oaiRHyISQQpqkhe9gEZ+iEg0KaiBN8wspzhuGvkhIpGkoAaK4zHm1uq2XCISTQrqwJWzKtl5+GTYZYiIXEBBHVg6p4ZXj53h+KmBsEsRETmPgjqwdE41AG2HToRciYjI+RTUgVRQb29XUItItCioA9XlxVwxs5y2gwpqEYkWBfUwSxurdUQtIpGjoB5m6ZxqDnaf4ZhOKIpIhCioh7k66Kd+Ud0fIhIhCuphFqdGfiioRSRCFNTDVE8rpqm2nO3t3WGXIiJyjoJ6hKWNNbQd1BWKIhIdCuoRls6ZzsHuMxzt7Q+7FBERQEF9gaVzagCdUBSR6FBQj7B4znQAXtR4ahGJCAX1CNPLiplfV6EjahGJDAX1KJbMqVZQi0hkKKhHcXVjNa+d6KOzRycURSR8CupRLNGFLyISIQrqUSy+fDpmmvJURKJh3KA2szeY2c/N7GUze8nMPjkVhYWpqqyYeTqhKCIRkc4R9SDwaXdfBKwEPm5mi7JbVviunlPNiwe7wy5DRGT8oHb319z9hWC5B9gBzMl2YWFbMqeaIyf76TjZF3YpIlLgJtRHbWZNwJuA50d57W4zazWz1s7OzgyVF56rG2sAXaEoIuFLO6jNrBJ4CviUu18wa5G7r3f35e6+vL6+PpM1hiJ1QlFBLSJhSyuozayYZEg/6u7fy25J0VBRWsSC+kpdSi4ioUtn1IcBDwE73P3vs19SdCzVFYoiEgHpHFGvAj4A3GJm24LHrVmuKxKWzqmmo6efIzqhKCIhKhpvA3ffBNgU1BI5SxuDeyi2n2DWorKQqxGRQqUrE8ewaPZ0Ygbb1f0hIiFSUI8hdUJRc36ISJgU1ONY2ljN9vYTuHvYpYhIgVJQj2PpnGq6evs5clJTnopIOBTU47g6OKG4vb073EJEpGApqMexaHY1MdPc1CISHgX1OKaVxGlpqNLIDxEJjYI6DUvmVNN2UCcURSQcCuo0XN1YTVfvAK+d0BWKIjL1FNRpSN1DUfN+iEgYFNRpWDR7OvGY8cKB42GXIiIFSEGdhmklcW5qqeOpLQfpHxwKuxwRKTAK6jR9ePU8unr7+eG2Q2GXIiIFRkGdptXNdVw1q4qHNr2i0R8iMqUU1GkyM/7j6iZ2Hu5h856jYZcjIgVEQT0B71o2h9qKEh7a9ErYpYhIAVFQT0BZcZz3r5zLT3d2sLezN+xyRKRAKKgn6AMr51ISj/HNZ/eFXYqIFAgF9QTVV5XyrmWX8+SWdrpPD4RdjogUAAX1JHz4pnmcOTvEY785EHYpIlIAFNST8MbLprOquZZv/3o/Z4cSYZcjInlOQT1JH149j8Mn+3j6xdfCLkVE8pyCepJuvrKB+fUVugBGRLJOQT1JsZjx56vmsb39BK37NVmTiGSPgvoSvPfaOVRPK+ahX+kCGBHJHgX1JSgvKeLPbriCDS8f5tVjp8MuR0TylIL6En3wzU3EzHQBjIhkjYL6El1WXcZtV8/mu62vcrLvbNjliEgeUlBnwEdWz+fUwCBrvvJLHnluv24uICIZpaDOgKWN1Tzy4RuYVV3G5/53G3/8d8/w6PP7GRjUxTAicuksG2OAly9f7q2trRnfb9S5O7/a1cVXNv6BrQe6mVMzjY//cTPvu66RkiL9TRSRizOzLe6+fNTXFNSZ5+78clcXX/nJH9j2ajKw/9MfzWdF00zm11dQWhQPu0QRiZixgrpoqospBGbGH11Zz1ta6s4F9r0/eAmAeMyYW1vOlQ1VXDmrkpZZVVw5q4qmunIFuIiMKq2gNrM1wNeAOPCgu38pq1XlieGB/Ycjvew8fJJdR3rZ1dHDH470sOHlwySG/UMzs6KEhqpS6qtKaagqo2F6KQ3Bcl1lCTXlJdSUF1M9rZiyYoW6SKEYN6jNLA78I/B2oB34rZn90N1fznZx+cLMuOqyKq66rOq89X1nh9jbeYpdHT3s6zpNR08fR07209nTx+6OXjp7+hlMjN41VVYco2ZaMrhryouZVhynOB4LHkZRsFwSt+S6ohgl8RilxcHXohglqUc8TjxmwQPisRhxM2IxKIrFzq0rCrYpOrftsIcZZsOWYyT3YYYZyQfBctAmya/JZRG5uHSOqK8Hdrv7XgAz+w7wLkBBfYnKiuMsunw6iy6fPurriYRz/PQAHT39HDs1QPfps3SfCb6eTj1PLnf1DnB2KBE8nMGhBANDzmAiwcDg6+ujygxiZsSGBfq5kOf1YCd4TmrdsOcMW39uOdg3BPsesf+J1JfaPPX+kdI53TPyfWNVkMk/YHbBwggT/dEYp7TxKh/rs031JGcTaefxaptZUcIT//nGSy3pAukE9Rzg1WHP24EbRm5kZncDdwNcccUVGSmu0MViRm1lKbWVpRnZXyLhDAwl6B9MhvfAUPB1MMFQwpMP99eXE07CncGEk0gkvw4lEsFXZ3Do/O3dU++BhCffO5RILkPyh9w9mQnJr8OfJ5eT73v9tUTCL9g+xd0vyJeRv0cJf/394CQSw/adZh44fi7IUrWmlsf6I3HBfkYUN9a3Hy+rRvveY2078vtPtPbzaxu7uHGbNZ12n+zfqIk0zCi1OI5dwl+h6WXZOe2Xsb26+3pgPSRHfWRqv5I5sZhRFourf1skx6QzuPcg8IZhzxuDdSIiMgXSCerfAi1mNs/MSoA7gR9mtywREUkZt+vD3QfN7BPA/yM5PO9hd38p65WJiAiQZh+1uz8NPJ3lWkREZBSagEJEJOIU1CIiEaegFhGJOAW1iEjEZWWaUzPrBPZP8u11QFcGy8kk1TY5qm1yVNvk5Gptc929frQXshLUl8LMWi82J2vYVNvkqLbJUW2Tk4+1qetDRCTiFNQiIhEXxaBeH3YBY1Btk6PaJke1TU7e1Ra5PmoRETlfFI+oRURkGAW1iEjERSaozWyNmf3ezHab2dqw6xnOzPaZ2Ytmts3MWiNQz8Nm1mFmbcPWzTSzn5jZruDrjAjV9gUzOxi03zYzuzWEut5gZj83s5fN7CUz+2SwPvR2G6O2KLRbmZn9xsx+F9R2X7B+npk9H/y+/lswBXJUavsXM3tlWLstm+rahtUYN7OtZvaj4Pnk2i15C6RwHySnT90DzAdKgN8Bi8Kua1h9+4C6sOsYVs9bgGuBtmHr/iewNlheC/xthGr7AvDfQm6z2cC1wXIV8AdgURTabYzaotBuBlQGy8XA88BK4LvAncH6fwY+FqHa/gV4X5jtNqzGvwQeA34UPJ9Uu0XliPrcDXTdfQBI3UBXRuHuvwSOjVj9LuBbwfK3gHdPZU0pF6ktdO7+mru/ECz3ADtI3g809HYbo7bQeVJv8LQ4eDhwC/BksD6sdrtYbZFgZo3AbcCDwXNjku0WlaAe7Qa6kfhBDTiwwcy2BDfxjaJZ7v5asHwYmBVmMaP4hJltD7pGQumWSTGzJuBNJI/AItVuI2qDCLRb8O/7NqAD+AnJ/3673X0w2CS039eRtbl7qt3+e9BuXzGzzNwdeuK+CvwVkAie1zLJdotKUEfdane/FvgT4ONm9pawCxqLJ/+visyRBfBPwAJgGfAa8L/CKsTMKoGngE+5+8nhr4XdbqPUFol2c/chd19G8n6p1wNvDKOO0YyszcyWAPeQrHEFMBP47FTXZWbvBDrcfUsm9heVoI70DXTd/WDwtQP4Pskf1qg5YmazAYKvHSHXc467Hwl+oRLAA4TUfmZWTDIIH3X37wWrI9Fuo9UWlXZLcfdu4OfAm4EaM0vdISr039dhta0JupLc3fuBbxJOu60CbjezfSS7cm8BvsYk2y0qQR3ZG+iaWYWZVaWWgXcAbWO/KxQ/BD4YLH8Q+EGItZwnFYSBf08I7Rf0Dz4E7HD3vx/2UujtdrHaItJu9WZWEyxPA95Osg/958D7gs3CarfRats57A+vkewDnvJ2c/d73L3R3ZtI5tnP3P39TLbdwj4rOuzs6K0kz3bvAf467HqG1TWf5CiU3wEvRaE24HGS/wqfJdnP9WGS/V8/BXYBG4GZEartX4EXge0kg3F2CHWtJtmtsR3YFjxujUK7jVFbFNrtamBrUEMbcG+wfj7wG2A38ARQGqHafha0WxvwCMHIkLAewM28PupjUu2mS8hFRCIuKl0fIiJyEQpqEZGIU1CLiEScglpEJOIU1CIiEaegFhnGzG5OzXQmEhUKahGRiFNQS04ys7uCuYi3mdk3gsl5eoNJeF4ys5+aWX2w7TIzey6YpOf7qcmNzKzZzDYG8xm/YGYLgt1XmtmTZrbTzB4NrnATCY2CWnKOmS0E/hRY5ckJeYaA9wMVQKu7LwZ+AawL3vJt4LPufjXJK9ZS6x8F/tHdrwFuJHlFJSRnr/sUyTmh55Oct0EkNEXjbyISOW8FrgN+GxzsTiM5mVIC+Ldgm0eA75lZNVDj7r8I1n8LeCKYv2WOu38fwN37AIL9/cbd24Pn24AmYFPWP5XIRSioJRcZ8C13v+e8lWafH7HdZOdH6B+2PIR+TyRk6vqQXPRT4H1m1gDn7ns4l+TPc2pmsj8DNrn7CeC4md0UrP8A8AtP3kml3czeHeyj1MzKp/JDiKRLRwqSc9z9ZTP7HMm77sRIztT3ceAUycnjP0eyK+RPg7d8EPjnIIj3An8erP8A8A0z+2Kwj/8whR9DJG2aPU/yhpn1untl2HWIZJq6PkREIk5H1CIiEacjahGRiFNQi4hEnIJaRCTiFNQiIhGnoBYRibj/Dz2aYk4smwLrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# fold 0: 19\n",
    "# fold 1: 34\n",
    "# fold 2: 34\n",
    "# fold 3: 40\n",
    "# avg: (19+34+34+40)/4 = 31.75\n",
    "# total epoch: 31.75 + 31.75*25/100 = 39.6875 = 40\n",
    "\n",
    "with strategy.scope():\n",
    "    model = effb5()\n",
    "    model.compile(\n",
    "        optimizer = tfa.optimizers.LAMB(LR),\n",
    "        loss = [tf.keras.losses.SparseCategoricalCrossentropy()],\n",
    "        metrics = [tf.keras.metrics.SparseCategoricalAccuracy()]\n",
    "    )\n",
    "model.summary()\n",
    "    \n",
    "model.fit(\n",
    "    train_data, \n",
    "    steps_per_epoch=train_size//BATCH_SIZE,\n",
    "    epochs = 40,\n",
    "    callbacks=[get_lr_callback()], verbose=1\n",
    ")\n",
    "\n",
    "h = model.history\n",
    "plt_acc(h)\n",
    "plt_loss(h)\n",
    "\n",
    "model.save_weights('model_weights.h5')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 19649.694987,
   "end_time": "2021-05-05T14:07:04.535621",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-05-05T08:39:34.840634",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}