final-solution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "determined-penny",
   "metadata": {
    "papermill": {
     "duration": 0.050147,
     "end_time": "2021-06-15T03:15:25.611666",
     "exception": false,
     "start_time": "2021-06-15T03:15:25.561519",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Import all libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "finnish-tiger",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:15:25.725542Z",
     "iopub.status.busy": "2021-06-15T03:15:25.724773Z",
     "iopub.status.idle": "2021-06-15T03:16:32.098969Z",
     "shell.execute_reply": "2021-06-15T03:16:32.098319Z",
     "shell.execute_reply.started": "2021-06-15T03:04:59.786671Z"
    },
    "papermill": {
     "duration": 66.440691,
     "end_time": "2021-06-15T03:16:32.099173",
     "exception": false,
     "start_time": "2021-06-15T03:15:25.658482",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install --quiet ../input/keras-efficientnet-whl/Keras_Applications-1.0.8-py3-none-any.whl\n",
    "!pip install --quiet ../input/keras-efficientnet-whl/efficientnet-1.1.1-py3-none-any.whl\n",
    "\n",
    "import sys\n",
    "sys.path.append('../input/nfnets-keras')\n",
    "\n",
    "import os\n",
    "import math\n",
    "import re\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import tensorflow_addons as tfa\n",
    "import numpy as np\n",
    "import tensorflow.keras.backend as K\n",
    "import efficientnet.keras as efn\n",
    "import efficientnet\n",
    "import itertools\n",
    "import matplotlib\n",
    "import scipy\n",
    "import pandas as pd\n",
    "import sklearn\n",
    "from matplotlib import pyplot as plt\n",
    "from datetime import datetime\n",
    "from functools import partial\n",
    "from kaggle_datasets import KaggleDatasets\n",
    "import pickle\n",
    "from collections import Counter\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "## for bert language model\n",
    "import transformers\n",
    "from transformers import TFAutoModel, AutoTokenizer\n",
    "from transformers import RobertaTokenizer, TFRobertaModel\n",
    "from nfnet import NFNet, nfnet_params\n",
    "\n",
    "import cudf\n",
    "import cuml\n",
    "import cupy\n",
    "from cuml.feature_extraction.text import TfidfVectorizer\n",
    "from cuml.neighbors import NearestNeighbors\n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "import string\n",
    "from tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n",
    "\n",
    "import gc\n",
    "\n",
    "SEED = 42\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)\n",
    "tf.random.set_seed(SEED)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "quick-portable",
   "metadata": {
    "papermill": {
     "duration": 0.027546,
     "end_time": "2021-06-15T03:16:32.154112",
     "exception": false,
     "start_time": "2021-06-15T03:16:32.126566",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Memory adjustment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "blessed-association",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:32.228326Z",
     "iopub.status.busy": "2021-06-15T03:16:32.227672Z",
     "iopub.status.idle": "2021-06-15T03:16:44.035720Z",
     "shell.execute_reply": "2021-06-15T03:16:44.034924Z",
     "shell.execute_reply.started": "2021-06-15T03:06:09.516596Z"
    },
    "papermill": {
     "duration": 11.854971,
     "end_time": "2021-06-15T03:16:44.035890",
     "exception": false,
     "start_time": "2021-06-15T03:16:32.180919",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We will restrict TensorFlow to max 5GB GPU RAM\n",
      "then RAPIDS can use 11GB GPU RAM\n"
     ]
    }
   ],
   "source": [
    "# RESTRICT TENSORFLOW TO 2GB OF GPU RAM\n",
    "# SO THAT WE HAVE 14GB RAM FOR RAPIDS\n",
    "LIMIT = 5.0\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "    try:\n",
    "        tf.config.experimental.set_virtual_device_configuration(\n",
    "            gpus[0],\n",
    "            [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=1024*LIMIT)])\n",
    "        logical_gpus = tf.config.experimental.list_logical_devices('GPU')\n",
    "        #print(len(gpus), \"Physical GPUs,\", len(logical_gpus), \"Logical GPUs\")\n",
    "    except RuntimeError as e:\n",
    "        print(e)\n",
    "print('We will restrict TensorFlow to max %iGB GPU RAM'%LIMIT)\n",
    "print('then RAPIDS can use %iGB GPU RAM'%(16-LIMIT))\n",
    "\n",
    "AUTO = tf.data.experimental.AUTOTUNE\n",
    "BATCH_SIZE = 32\n",
    "IMAGE_SIZE = (380, 380)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "opponent-prior",
   "metadata": {
    "papermill": {
     "duration": 0.045749,
     "end_time": "2021-06-15T03:16:44.128075",
     "exception": false,
     "start_time": "2021-06-15T03:16:44.082326",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Help functions for text preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "explicit-fabric",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:44.239950Z",
     "iopub.status.busy": "2021-06-15T03:16:44.237289Z",
     "iopub.status.idle": "2021-06-15T03:16:44.256684Z",
     "shell.execute_reply": "2021-06-15T03:16:44.257771Z",
     "shell.execute_reply.started": "2021-06-15T03:06:22.515359Z"
    },
    "papermill": {
     "duration": 0.084792,
     "end_time": "2021-06-15T03:16:44.257947",
     "exception": false,
     "start_time": "2021-06-15T03:16:44.173155",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Preprocessing function helper\n",
    "# replace word that concatenate with other word\n",
    "def remove_concatenate_2_words(text):\n",
    "    list_words = ['khusus']\n",
    "    for w in list_words:\n",
    "        text = text.replace(w, '')\n",
    "    return text\n",
    "\n",
    "PUNCT_TO_REMOVE = string.punctuation\n",
    "def remove_punctuation(text):\n",
    "    return text.translate(str.maketrans('', '', PUNCT_TO_REMOVE))\n",
    "\n",
    "STOPWORDS_ID = set(stopwords.words('indonesian'))\n",
    "STOPWORDS_EN = set(stopwords.words('english'))\n",
    "def remove_stopwords(list_text):\n",
    "    text_not_in_ID = [word for word in list_text if word not in STOPWORDS_EN]\n",
    "    text = [word for word in text_not_in_ID if word not in STOPWORDS_ID]\n",
    "    return text\n",
    "\n",
    "# remove big number and split text that contains word and number\n",
    "def remove_big_number(list_text):\n",
    "    words = []\n",
    "    for w in list_text:\n",
    "        sub_w = re.split('(\\d+)',w)\n",
    "        for item in sub_w:\n",
    "            try:\n",
    "                tmp = int(item)\n",
    "                if tmp < 7000:\n",
    "                    if (tmp>1000) and (tmp % 100 == 0): # for even number\n",
    "                        words.append(str(tmp))\n",
    "                    elif (tmp<=1000) and (tmp>100) and (tmp % 10 == 0 ):\n",
    "                        words.append(str(tmp))\n",
    "                    elif (tmp<=100) and (tmp % 2 == 0):\n",
    "                        words.append(str(tmp))\n",
    "            except:\n",
    "                words.append(item)\n",
    "    return words\n",
    "\n",
    "def remove_zero_val(list_text):\n",
    "    return [w for w in list_text if w not in ['0']]\n",
    "\n",
    "def remove_common_words(list_text):\n",
    "    common_words = \"hari keren kere kw super baik jual jualan quality best free  kwalitas berkualitas kualitas bagus terbaik kembali dijamin beli gratis murah free diskon ongkir cek berkualitas original asli kualitas uang jaminan jamin terjamin buatan buat kirim wilayah luar kota jawa bali jakarta surabaya bulan month year day tahun hari harian anda your nikmat singapore malaysia indonesia vietnam thailand filipina bangkok jepang buy one get dapat dua two satu meriah kirim send pengiriman paket hemat uang kembali dapat guarantee buatan lokal dalam internasional karya termurah paling murah terbaik cheap murah biaya\".split(' ')\n",
    "    return [w for w in list_text if w not in common_words]\n",
    "\n",
    "def remove_strange_words(list_text):\n",
    "    strange_words = ['aaa', 'aaaa', 'aaaaa', 'abc', 'abcd', 'bb', 'bbb', 'bbbb', 'ccc', 'cccc', 'thn', 'th', 'bln']\n",
    "    return [w for w in list_text if w not in strange_words]\n",
    "\n",
    "def text_vectorizer(max_features, max_len, vocab):\n",
    "    # max_features: Maximum vocab size.\n",
    "    # max_len: Sequence length to pad the outputs to.\n",
    "    \n",
    "    text_dataset = tf.data.Dataset.from_tensor_slices(vocab)\n",
    "    \n",
    "    # Create the layer.\n",
    "    vectorize_layer = TextVectorization(\n",
    "        max_tokens = max_features,\n",
    "        output_mode = 'int',\n",
    "        output_sequence_length = max_len\n",
    "    )\n",
    "\n",
    "    vectorize_layer.adapt(text_dataset.batch(64))\n",
    "\n",
    "    model = tf.keras.models.Sequential()\n",
    "    model.add(tf.keras.Input(shape=(1,), dtype=tf.string))\n",
    "    model.add(vectorize_layer)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "light-sucking",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:44.367706Z",
     "iopub.status.busy": "2021-06-15T03:16:44.366242Z",
     "iopub.status.idle": "2021-06-15T03:16:44.368556Z",
     "shell.execute_reply": "2021-06-15T03:16:44.368991Z",
     "shell.execute_reply.started": "2021-06-15T03:06:24.016722Z"
    },
    "papermill": {
     "duration": 0.063053,
     "end_time": "2021-06-15T03:16:44.369159",
     "exception": false,
     "start_time": "2021-06-15T03:16:44.306106",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def utils_preprocess_text(text, flg_stemm=False, flg_lemm=True, lst_stopwords=None):\n",
    "    ## clean (convert to lowercase and remove punctuations and characters and then strip\n",
    "    text = re.sub(r'[^\\w\\s]', '', str(text).lower().strip())\n",
    "            \n",
    "    ## Tokenize (convert from string to list)\n",
    "    lst_text = text.split()\n",
    "    ## remove Stopwords\n",
    "    if lst_stopwords is not None:\n",
    "        for stopwords in lst_stopwords:\n",
    "            lst_text = [word for word in lst_text if word not in \n",
    "                        stopwords]\n",
    "                \n",
    "    ## Stemming (remove -ing, -ly, ...)\n",
    "    if flg_stemm == True:\n",
    "        # english stemming\n",
    "        ps = nltk.stem.porter.PorterStemmer()\n",
    "        lst_text = [ps.stem(word) for word in lst_text]\n",
    "        \n",
    "        # indonesian stemming\n",
    "#         factory = StemmerFactory()\n",
    "#         id_stemmer = factory.create_stemmer()\n",
    "\n",
    "#         lst_text = [id_stemmer.stem(word) for word in lst_text]\n",
    "                \n",
    "    ## Lemmatisation (convert the word into root word)\n",
    "    if flg_lemm == True:\n",
    "        lem = nltk.stem.wordnet.WordNetLemmatizer()\n",
    "        lst_text = [lem.lemmatize(word) for word in lst_text]\n",
    "        \n",
    "    # remove_zero_val\n",
    "    lst_text = [w for w in lst_text if w not in ['0']]\n",
    "    \n",
    "    # remove strange words\n",
    "    strange_words = ['aaa', 'aaaa', 'aaaaa', 'abc', 'abcd', 'bb', 'bbb', 'bbbb', 'ccc', 'cccc', 'thn', 'th', 'bln']\n",
    "    lst_text = [w for w in lst_text if w not in strange_words]\n",
    "            \n",
    "    ## back to string from list\n",
    "    text = \" \".join(lst_text)\n",
    "    return text\n",
    "\n",
    "def string_escape(s, encoding='utf-8'):\n",
    "    return (\n",
    "        s.encode('latin1')  # To bytes, required by 'unicode-escape'\n",
    "        .decode('unicode-escape')  # Perform the actual octal-escaping decode\n",
    "        .encode('latin1')  # 1:1 mapping back to bytes\n",
    "        .decode(encoding)\n",
    "    )  # Decode original encoding\n",
    "\n",
    "def regular_encode(texts, tokenizer, maxlen=512):\n",
    "    enc_di = tokenizer.batch_encode_plus(\n",
    "        texts, \n",
    "#         add_special_tokens = True,\n",
    "        return_attention_mask = True,\n",
    "        return_token_type_ids=True,\n",
    "        pad_to_max_length=True,\n",
    "        max_length=maxlen\n",
    "        )\n",
    "    \n",
    "    return np.array(enc_di['input_ids']), np.array(enc_di['attention_mask'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "educational-evanescence",
   "metadata": {
    "papermill": {
     "duration": 0.027329,
     "end_time": "2021-06-15T03:16:44.423646",
     "exception": false,
     "start_time": "2021-06-15T03:16:44.396317",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Title preprocessing (train vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "pleased-bowling",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:44.487804Z",
     "iopub.status.busy": "2021-06-15T03:16:44.487296Z",
     "iopub.status.idle": "2021-06-15T03:16:47.189622Z",
     "shell.execute_reply": "2021-06-15T03:16:47.188709Z",
     "shell.execute_reply.started": "2021-06-15T03:11:08.211321Z"
    },
    "papermill": {
     "duration": 2.738664,
     "end_time": "2021-06-15T03:16:47.189767",
     "exception": false,
     "start_time": "2021-06-15T03:16:44.451103",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# extract vocab from train data\n",
    "df = pd.read_csv('../input/shopee-product-matching/train.csv')\n",
    "    \n",
    "df['title'] = df['title'].apply(lambda x: string_escape(x))\n",
    "df['title'] = df['title'].apply(lambda x: remove_concatenate_2_words(x))\n",
    "df['title'] = df['title'].str.lower()\n",
    "df['title'] = df['title'].apply(lambda x: remove_punctuation(x))\n",
    "df['title'] = df['title'].apply(lambda x: str(x).split())\n",
    "df['title'] = df['title'].apply(lambda x: remove_stopwords(x))\n",
    "# df['title'] = df['title'].apply(lambda x: remove_big_number(x))\n",
    "df['title'] = df['title'].apply(lambda x: remove_zero_val(x))\n",
    "df['title'] = df['title'].apply(lambda x: remove_common_words(x))\n",
    "df['title'] = df['title'].apply(lambda x: remove_strange_words(x))\n",
    "df['title'] = df['title'].apply(lambda x: list(np.unique(x)))\n",
    "\n",
    "# title vocab\n",
    "words = list(df['title'])\n",
    "train_vocab = list(np.unique(np.concatenate(words)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "blond-processing",
   "metadata": {
    "papermill": {
     "duration": 0.027443,
     "end_time": "2021-06-15T03:16:47.244829",
     "exception": false,
     "start_time": "2021-06-15T03:16:47.217386",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Load test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "earned-convertible",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:47.309294Z",
     "iopub.status.busy": "2021-06-15T03:16:47.307786Z",
     "iopub.status.idle": "2021-06-15T03:16:47.317387Z",
     "shell.execute_reply": "2021-06-15T03:16:47.316901Z",
     "shell.execute_reply.started": "2021-06-15T03:11:10.704884Z"
    },
    "papermill": {
     "duration": 0.043977,
     "end_time": "2021-06-15T03:16:47.317493",
     "exception": false,
     "start_time": "2021-06-15T03:16:47.273516",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "GET_CV = False\n",
    "CHECK_RAM = False\n",
    "\n",
    "if GET_CV:\n",
    "    if CHECK_RAM:\n",
    "        df = pd.read_csv('../input/shopee-product-matching/train.csv')\n",
    "        df = pd.concat([df,df], axis=0)\n",
    "    else:\n",
    "        df = pd.read_parquet('../input/shopee-tfrecords-380-gkf-four-folds/fold_3/unseen.parquet', engine='pyarrow')\n",
    "else:\n",
    "    df = pd.read_csv('../input/shopee-product-matching/test.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "lovely-deviation",
   "metadata": {
    "papermill": {
     "duration": 0.027486,
     "end_time": "2021-06-15T03:16:47.372499",
     "exception": false,
     "start_time": "2021-06-15T03:16:47.345013",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Preprocessing for MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "skilled-companion",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:47.440038Z",
     "iopub.status.busy": "2021-06-15T03:16:47.434729Z",
     "iopub.status.idle": "2021-06-15T03:16:48.948049Z",
     "shell.execute_reply": "2021-06-15T03:16:48.947248Z",
     "shell.execute_reply.started": "2021-06-15T03:11:10.719217Z"
    },
    "papermill": {
     "duration": 1.548255,
     "end_time": "2021-06-15T03:16:48.948182",
     "exception": false,
     "start_time": "2021-06-15T03:16:47.399927",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>posting_id</th>\n",
       "      <th>image</th>\n",
       "      <th>image_phash</th>\n",
       "      <th>title</th>\n",
       "      <th>title_vec</th>\n",
       "      <th>input_ids</th>\n",
       "      <th>att_mask</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test_2255846744</td>\n",
       "      <td>0006c8e5462ae52167402bac1c2e916e.jpg</td>\n",
       "      <td>ecc292392dc7687a</td>\n",
       "      <td>Edufuntoys - CHARACTER PHONE ada lampu dan mus...</td>\n",
       "      <td>[21, 20, 13, 11, 7, 6, 2, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test_3588702337</td>\n",
       "      <td>0007585c4d0f932859339129f709bfdc.jpg</td>\n",
       "      <td>e9968f60d2699e2c</td>\n",
       "      <td>(Beli 1 Free Spatula) Masker Komedo | Blackhea...</td>\n",
       "      <td>[24, 23, 22, 19, 17374, 18, 15, 10, 9, 3, 0, 0...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>test_4015706929</td>\n",
       "      <td>0008377d3662e83ef44e1881af38b879.jpg</td>\n",
       "      <td>ba81c17e3581cabe</td>\n",
       "      <td>READY Lemonilo Mie instant sehat kuah dan goreng</td>\n",
       "      <td>[17, 16, 14, 12, 8, 5, 4, 0, 0, 0, 0, 0, 0, 0,...</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        posting_id                                 image       image_phash  \\\n",
       "0  test_2255846744  0006c8e5462ae52167402bac1c2e916e.jpg  ecc292392dc7687a   \n",
       "1  test_3588702337  0007585c4d0f932859339129f709bfdc.jpg  e9968f60d2699e2c   \n",
       "2  test_4015706929  0008377d3662e83ef44e1881af38b879.jpg  ba81c17e3581cabe   \n",
       "\n",
       "                                               title  \\\n",
       "0  Edufuntoys - CHARACTER PHONE ada lampu dan mus...   \n",
       "1  (Beli 1 Free Spatula) Masker Komedo | Blackhea...   \n",
       "2   READY Lemonilo Mie instant sehat kuah dan goreng   \n",
       "\n",
       "                                           title_vec input_ids att_mask  \n",
       "0  [21, 20, 13, 11, 7, 6, 2, 0, 0, 0, 0, 0, 0, 0,...      None     None  \n",
       "1  [24, 23, 22, 19, 17374, 18, 15, 10, 9, 3, 0, 0...      None     None  \n",
       "2  [17, 16, 14, 12, 8, 5, 4, 0, 0, 0, 0, 0, 0, 0,...      None     None  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "################ for Image + MLP ################\n",
    "# title preprocessing for test dataset\n",
    "df['tmp'] = df['title'].apply(lambda x: string_escape(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_concatenate_2_words(x))\n",
    "df['tmp'] = df['tmp'].str.lower()\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_punctuation(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: str(x).split())\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_stopwords(x))\n",
    "# df['tmp'] = df['tmp'].apply(lambda x: remove_big_number(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_zero_val(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_common_words(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_strange_words(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: list(np.unique(x)))\n",
    "\n",
    "# title vocab\n",
    "words = list(df['tmp'])\n",
    "words = list(np.unique(np.concatenate(words)))\n",
    "words = train_vocab + words\n",
    "\n",
    "# Text vectorizer\n",
    "model = text_vectorizer(max_features = 25000, max_len = 100, vocab = words)\n",
    "list_text = [' '.join(x) for x in df['tmp']]\n",
    "title_vec = model.predict(list_text)\n",
    "df['title_vec'] = list(title_vec)\n",
    "df['input_ids'] = None\n",
    "df['att_mask'] = None\n",
    "\n",
    "del words, model, list_text, title_vec, df['tmp'], train_vocab\n",
    "gc.collect()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "visible-assault",
   "metadata": {
    "papermill": {
     "duration": 0.028187,
     "end_time": "2021-06-15T03:16:49.005025",
     "exception": false,
     "start_time": "2021-06-15T03:16:48.976838",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Help functions for datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "popular-basket",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:49.084413Z",
     "iopub.status.busy": "2021-06-15T03:16:49.082499Z",
     "iopub.status.idle": "2021-06-15T03:16:49.085143Z",
     "shell.execute_reply": "2021-06-15T03:16:49.085634Z",
     "shell.execute_reply.started": "2021-06-15T03:06:48.693555Z"
    },
    "papermill": {
     "duration": 0.051911,
     "end_time": "2021-06-15T03:16:49.085790",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.033879",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def getMetric(col):\n",
    "    def f1score(row):\n",
    "        n = len( np.intersect1d(row.target,row[col]) )\n",
    "        return 2*n / (len(row.target)+len(row[col]))\n",
    "    return f1score\n",
    "\n",
    "def read_dataset(df):\n",
    "    if GET_CV:\n",
    "        image_paths = '/kaggle/input/shopee-product-matching/train_images/' + df['image']\n",
    "    else:\n",
    "        image_paths = '/kaggle/input/shopee-product-matching/test_images/' + df['image']\n",
    "    input_ids = np.stack(df['input_ids'], axis=0)\n",
    "    att_mask = np.stack(df['att_mask'], axis=0)\n",
    "    title_vec = np.stack(df['title_vec'], axis=0)\n",
    "    return image_paths, input_ids, att_mask, title_vec\n",
    "\n",
    "def decode_image(image, img_size):\n",
    "    image = tf.image.decode_jpeg(image, channels=3)\n",
    "    image = tf.image.resize(image, img_size)\n",
    "    image = tf.cast(image, tf.float32) / 255.0\n",
    "    return tf.reshape(image, [*img_size, 3])\n",
    "\n",
    "# Function to read our test image and return image\n",
    "def read_image(filename, title_vec):\n",
    "    image = tf.io.read_file(filename)\n",
    "    image = decode_image(image, IMAGE_SIZE)\n",
    "    if ONLY_IMAGE:\n",
    "        return (image), np.empty((0), dtype=int)\n",
    "    else:\n",
    "        return (image, title_vec), np.empty((0), dtype=int)\n",
    "\n",
    "# Function to get our dataset that read images\n",
    "def get_dataset(image, title_vec):\n",
    "    dataset = tf.data.Dataset.from_tensor_slices((image, title_vec))\n",
    "    dataset = dataset.map(read_image, num_parallel_calls = AUTO)\n",
    "    dataset = dataset.batch(BATCH_SIZE)\n",
    "    dataset = dataset.prefetch(AUTO)\n",
    "    return dataset\n",
    "\n",
    "def get_text_dataset(input_ids, att_mask, title_vec):\n",
    "    dataset = tf.data.Dataset.from_tensor_slices((input_ids, att_mask, title_vec))\n",
    "    dataset = dataset.map(lambda x,y,z: ((x,y,z), ()), num_parallel_calls = AUTO)\n",
    "    dataset = dataset.batch(BATCH_SIZE)\n",
    "    dataset = dataset.prefetch(AUTO)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "instrumental-farmer",
   "metadata": {
    "papermill": {
     "duration": 0.030407,
     "end_time": "2021-06-15T03:16:49.147018",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.116611",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Import library for Deep Learning Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fantastic-canon",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:49.215567Z",
     "iopub.status.busy": "2021-06-15T03:16:49.214746Z",
     "iopub.status.idle": "2021-06-15T03:16:49.221525Z",
     "shell.execute_reply": "2021-06-15T03:16:49.220989Z",
     "shell.execute_reply.started": "2021-06-15T03:06:50.349762Z"
    },
    "papermill": {
     "duration": 0.045017,
     "end_time": "2021-06-15T03:16:49.221652",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.176635",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Model, Sequential\n",
    "import keras.backend as K\n",
    "from keras.optimizers import SGD\n",
    "from tensorflow.keras.layers import (\n",
    "    Input, \n",
    "    Flatten, \n",
    "    Dense, \n",
    "    Dropout, \n",
    "    AveragePooling2D, \n",
    "    GlobalAveragePooling2D, \n",
    "    SpatialDropout2D, \n",
    "    BatchNormalization, \n",
    "    Activation, \n",
    "    Concatenate,\n",
    "    Embedding,\n",
    "    GlobalAveragePooling1D,\n",
    "    Lambda\n",
    ")\n",
    "\n",
    "from keras.backend import sigmoid\n",
    "def swish(x, beta = 1):\n",
    "    return (x * sigmoid(beta * x))\n",
    "\n",
    "from keras.utils.generic_utils import get_custom_objects\n",
    "from keras.layers import Activation\n",
    "\n",
    "get_custom_objects().update({'swish': Activation(swish)})\n",
    "\n",
    "from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, multiply, Reshape\n",
    "from keras.layers import Bidirectional, GlobalMaxPool1D,Bidirectional, Conv1D, GlobalMaxPooling1D, Conv2D\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.applications import ResNet50, InceptionResNetV2, Xception"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "arbitrary-biology",
   "metadata": {
    "papermill": {
     "duration": 0.03162,
     "end_time": "2021-06-15T03:16:49.286755",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.255135",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Arc margin implementation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "intermediate-irish",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:49.357829Z",
     "iopub.status.busy": "2021-06-15T03:16:49.356617Z",
     "iopub.status.idle": "2021-06-15T03:16:49.358876Z",
     "shell.execute_reply": "2021-06-15T03:16:49.359344Z",
     "shell.execute_reply.started": "2021-06-15T03:06:52.060690Z"
    },
    "papermill": {
     "duration": 0.044068,
     "end_time": "2021-06-15T03:16:49.359473",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.315405",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ArcMarginProduct(keras.layers.Layer):\n",
    "    '''\n",
    "    Implements large margin arc distance.\n",
    "\n",
    "    Reference:\n",
    "        https://arxiv.org/pdf/1801.07698.pdf\n",
    "        https://github.com/lyakaap/Landmark2019-1st-and-3rd-Place-Solution/\n",
    "            blob/master/src/modeling/metric_learning.py\n",
    "    '''\n",
    "    def __init__(self, n_classes, s=30, m=0.50, easy_margin=False,\n",
    "                 ls_eps=0.0, **kwargs):\n",
    "\n",
    "        super(ArcMarginProduct, self).__init__(**kwargs)\n",
    "\n",
    "        self.n_classes = n_classes\n",
    "        self.s = s\n",
    "        self.m = m\n",
    "        self.ls_eps = ls_eps\n",
    "        self.easy_margin = easy_margin\n",
    "        self.cos_m = tf.math.cos(m)\n",
    "        self.sin_m = tf.math.sin(m)\n",
    "        self.th = tf.math.cos(math.pi - m)\n",
    "        self.mm = tf.math.sin(math.pi - m) * m\n",
    "\n",
    "    def get_config(self):\n",
    "\n",
    "        config = super().get_config().copy()\n",
    "        config.update({\n",
    "            'n_classes': self.n_classes,\n",
    "            's': self.s,\n",
    "            'm': self.m,\n",
    "            'ls_eps': self.ls_eps,\n",
    "            'easy_margin': self.easy_margin,\n",
    "        })\n",
    "        return config\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        super(ArcMarginProduct, self).build(input_shape[0])\n",
    "\n",
    "        self.W = self.add_weight(\n",
    "            name='W',\n",
    "            shape=(int(input_shape[0][-1]), self.n_classes),\n",
    "            initializer='glorot_uniform',\n",
    "            dtype='float32',\n",
    "            trainable=True,\n",
    "            regularizer=None)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        X, y = inputs\n",
    "        y = tf.cast(y, dtype=tf.int32)\n",
    "        cosine = tf.matmul(\n",
    "            tf.math.l2_normalize(X, axis=1),\n",
    "            tf.math.l2_normalize(self.W, axis=0)\n",
    "        )\n",
    "        sine = tf.math.sqrt(1.0 - tf.math.pow(cosine, 2))\n",
    "        phi = cosine * self.cos_m - sine * self.sin_m\n",
    "        if self.easy_margin:\n",
    "            phi = tf.where(cosine > 0, phi, cosine)\n",
    "        else:\n",
    "            phi = tf.where(cosine > self.th, phi, cosine - self.mm)\n",
    "        one_hot = tf.cast(\n",
    "            tf.one_hot(y, depth=self.n_classes),\n",
    "            dtype=cosine.dtype\n",
    "        )\n",
    "        if self.ls_eps > 0:\n",
    "            one_hot = (1 - self.ls_eps) * one_hot + self.ls_eps / self.n_classes\n",
    "\n",
    "        output = (one_hot * phi) + ((1.0 - one_hot) * cosine)\n",
    "        output *= self.s\n",
    "        return output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "martial-ferry",
   "metadata": {
    "papermill": {
     "duration": 0.027877,
     "end_time": "2021-06-15T03:16:49.416173",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.388296",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Deep learning models (Image + MLP for title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "funded-stopping",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:49.503518Z",
     "iopub.status.busy": "2021-06-15T03:16:49.482625Z",
     "iopub.status.idle": "2021-06-15T03:16:49.562968Z",
     "shell.execute_reply": "2021-06-15T03:16:49.562439Z",
     "shell.execute_reply.started": "2021-06-15T03:06:54.055697Z"
    },
    "papermill": {
     "duration": 0.119119,
     "end_time": "2021-06-15T03:16:49.563149",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.444030",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def effb1(weights='noisy-student'):\n",
    "    efn1 = efn.EfficientNetB1(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    \n",
    "    for layer in efn1.layers:\n",
    "        layer.trainable = False\n",
    "    \n",
    "    model_image = Sequential([\n",
    "        efn1,\n",
    "        GlobalAveragePooling2D(name='effb1-pooling'),\n",
    "        BatchNormalization(name='effb1_bn1'),\n",
    "        Dropout(0.2),\n",
    "        Dense(2400, name='effb1_dense1'),\n",
    "        Activation('swish', name='effb1_act1'),\n",
    "    ], name='effb1-img')\n",
    "    \n",
    "    eff_aux = Model(\n",
    "        inputs = efn1.input, \n",
    "        outputs = efn1.get_layer('block5b_activation').output)\n",
    "    aux_model = Sequential([\n",
    "        eff_aux,\n",
    "        Conv2D(128, kernel_size=(3, 3), activation='relu', name='aux-conv1'),\n",
    "        BatchNormalization(name='aux-bn1'),\n",
    "        GlobalAveragePooling2D(name='aux-pooling'),\n",
    "        # Flatten(),\n",
    "        Dropout(0.5),\n",
    "        Dense(1024, name='aux-dense1'),\n",
    "        Activation('swish'),\n",
    "    ], name='aux-model')\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 70, input_length=100, name='title-embed'),\n",
    "#         GlobalAveragePooling1D(name='title-pooling'),\n",
    "        Flatten(name='title-flatten'),\n",
    "        BatchNormalization(name='title-bn1'),\n",
    "        Dropout(0.2),\n",
    "        Dense(650, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "#         Dropout(0.1),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([model_image.output, model_title.output])\n",
    "    concatenate2 = Concatenate(name='concatenate2')([concatenate, aux_model.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate2, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[model_image.input, aux_model.input, model_title.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "def effb1_512_v2(weights='noisy-student'):\n",
    "    efn1 = efn.EfficientNetB1(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    \n",
    "    for layer in efn1.layers:\n",
    "        layer.trainable = False\n",
    "    \n",
    "    model_image = Sequential([\n",
    "        efn1,\n",
    "        GlobalAveragePooling2D(name='effb1-pooling'),\n",
    "        BatchNormalization(name='effb1_bn1'),\n",
    "        Dropout(0.2),\n",
    "        Dense(2400, name='effb1_dense1'),\n",
    "        Activation('swish', name='effb1_act1'),\n",
    "    ], name='effb1-img')\n",
    "    \n",
    "    eff_aux = Model(\n",
    "        inputs = efn1.input, \n",
    "        outputs = efn1.get_layer('block5b_activation').output)\n",
    "    aux_model = Sequential([\n",
    "        eff_aux,\n",
    "        Conv2D(128, kernel_size=(3, 3), activation='relu', name='aux-conv1'),\n",
    "        BatchNormalization(name='aux-bn1'),\n",
    "        GlobalAveragePooling2D(name='aux-pooling'),\n",
    "        Dropout(0.5),\n",
    "        Dense(1024, name='aux-dense1'),\n",
    "        Activation('swish'),\n",
    "    ], name='aux-model')\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([model_image.output, model_title.output])\n",
    "    concatenate2 = Concatenate(name='concatenate2')([concatenate, aux_model.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate2, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[model_image.input, aux_model.input, model_title.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "def effb1_512_v3(weights='noisy-student'):\n",
    "    efn1 = efn.EfficientNetB1(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    \n",
    "    model_image = Sequential([\n",
    "        efn1,\n",
    "        GlobalAveragePooling2D(name='effb1-pooling'),\n",
    "        BatchNormalization(name='effb1_bn1'),\n",
    "        Dropout(0.2),\n",
    "        Dense(2400, name='effb1_dense1'),\n",
    "        Activation('swish', name='effb1_act1'),\n",
    "    ], name='effb1-img')\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([model_image.output, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[model_image.input, model_title.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "def effb2(weights='noisy-student'):\n",
    "    efn2 = efn.EfficientNetB2(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    \n",
    "    for layer in efn2.layers:\n",
    "        layer.trainable = False\n",
    "    \n",
    "    model_image = Sequential([\n",
    "        efn2,\n",
    "        GlobalAveragePooling2D(name='effb1-pooling'),\n",
    "        BatchNormalization(name='effb1_bn1'),\n",
    "        Dropout(0.2),\n",
    "        Dense(2400, name='effb1_dense1'),\n",
    "        Activation('swish', name='effb1_act1'),\n",
    "    ], name='effb1-img')\n",
    "    \n",
    "    eff_aux = Model(\n",
    "        inputs = efn2.input, \n",
    "        outputs = efn2.get_layer('block5b_activation').output)\n",
    "    aux_model = Sequential([\n",
    "        eff_aux,\n",
    "        Conv2D(128, kernel_size=(3, 3), activation='relu', name='aux-conv1'),\n",
    "        BatchNormalization(name='aux-bn1'),\n",
    "        GlobalAveragePooling2D(name='aux-pooling'),\n",
    "        # Flatten(),\n",
    "        Dropout(0.5),\n",
    "        Dense(1024, name='aux-dense1'),\n",
    "        Activation('swish'),\n",
    "    ], name='aux-model')\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = 8261, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate2 = Concatenate(name='concatenate2')([model_image.output, aux_model.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate2, label])\n",
    "    output = Dense(8261, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[model_image.input, aux_model.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "def effb5(weights='noisy-student'):\n",
    "    effb5 = efn.EfficientNetB5(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3),\n",
    "                                drop_connect_rate=0  # the hack\n",
    "                              )\n",
    "    \n",
    "    model_image = Sequential([\n",
    "        effb5,\n",
    "        GlobalAveragePooling2D(name='effb1-pooling'),\n",
    "        BatchNormalization(name='effb1_bn1'),\n",
    "        Dropout(0.2),\n",
    "        Dense(2400, name='effb1_dense1'),\n",
    "        Activation('swish', name='effb1_act1'),\n",
    "    ], name='effb1-img')\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([model_image.output, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[model_image.input, model_title.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "def effb7(weights='noisy-student'):\n",
    "    img_inp = Input(shape=(*IMAGE_SIZE, 3))\n",
    "    effb5 = efn.EfficientNetB7(weights=weights, input_shape=(*IMAGE_SIZE, 3), include_top=False)\n",
    "    \n",
    "    pt_depth = effb5.layers[-1].get_output_shape_at(0)[-1]\n",
    "    pt_features = effb5(img_inp)\n",
    "    bn_features = BatchNormalization()(pt_features)\n",
    "    \n",
    "    attn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(Dropout(0.5)(bn_features))\n",
    "    attn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(8, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(1, kernel_size = (1,1), padding = 'valid', activation = 'sigmoid')(attn_layer)\n",
    "    \n",
    "    up_c2_w = np.ones((1, 1, 1, pt_depth))\n",
    "    up_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', activation = 'linear', \n",
    "                   use_bias = False, weights = [up_c2_w])\n",
    "    up_c2.trainable = False\n",
    "    attn_layer = up_c2(attn_layer)\n",
    "    \n",
    "    mask_features = multiply([attn_layer, bn_features])\n",
    "    gap_features = GlobalAveragePooling2D()(mask_features)\n",
    "    gap_mask = GlobalAveragePooling2D()(attn_layer)\n",
    "    # to account for missing values from the attention model\n",
    "    gap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\n",
    "    gap_dr = Dropout(0.25)(gap)\n",
    "    img_embed = Dense(2400, activation = 'swish', name='img-embed')(gap_dr)\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "    \n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([img_embed, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[img_inp, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "def effb7_v2(weights='noisy-student'):\n",
    "    img_inp = Input(shape=(*IMAGE_SIZE, 3))\n",
    "    effb7 = efn.EfficientNetB7(weights=weights, input_shape=(*IMAGE_SIZE, 3), include_top=False)\n",
    "    \n",
    "    pt_depth = effb7.layers[-1].get_output_shape_at(0)[-1]\n",
    "    pt_features = effb7(img_inp)\n",
    "    bn_features = BatchNormalization()(pt_features)\n",
    "    \n",
    "    attn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(Dropout(0.5)(bn_features))\n",
    "    attn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(8, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(1, kernel_size = (1,1), padding = 'valid', activation = 'sigmoid')(attn_layer)\n",
    "    \n",
    "    up_c2_w = np.ones((1, 1, 1, pt_depth))\n",
    "    up_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', activation = 'linear', \n",
    "                   use_bias = False, weights = [up_c2_w])\n",
    "    up_c2.trainable = False\n",
    "    attn_layer = up_c2(attn_layer)\n",
    "    \n",
    "    mask_features = multiply([attn_layer, bn_features])\n",
    "    gap_features = GlobalAveragePooling2D()(mask_features)\n",
    "    gap_mask = GlobalAveragePooling2D()(attn_layer)\n",
    "    # to account for missing values from the attention model\n",
    "    gap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\n",
    "    gap_dr = Dropout(0.25)(gap)\n",
    "    img_embed = Dense(2400, activation = 'swish')(gap_dr)\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(22000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([img_embed, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[img_inp, model_title.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "def effb5_v2(weights='noisy-student'):\n",
    "    img_inp = Input(shape=(*IMAGE_SIZE, 3))\n",
    "    effb5 = efn.EfficientNetB5(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    \n",
    "    pt_depth = effb5.layers[-1].get_output_shape_at(0)[-1]\n",
    "    pt_features = effb5(img_inp)\n",
    "    bn_features = BatchNormalization()(pt_features)\n",
    "    \n",
    "    attn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(Dropout(0.5)(bn_features))\n",
    "    attn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(8, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(1, kernel_size = (1,1), padding = 'valid', activation = 'sigmoid')(attn_layer)\n",
    "    \n",
    "    up_c2_w = np.ones((1, 1, 1, pt_depth))\n",
    "    up_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', activation = 'linear', \n",
    "                   use_bias = False, weights = [up_c2_w])\n",
    "    up_c2.trainable = False\n",
    "    attn_layer = up_c2(attn_layer)\n",
    "    \n",
    "    mask_features = multiply([attn_layer, bn_features])\n",
    "    gap_features = GlobalAveragePooling2D()(mask_features)\n",
    "    gap_mask = GlobalAveragePooling2D()(attn_layer)\n",
    "    # to account for missing values from the attention model\n",
    "    gap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\n",
    "    gap_dr = Dropout(0.25)(gap)\n",
    "    img_embed = Dense(2400, activation = 'swish', name='img-embed')(gap_dr)\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([img_embed, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[img_inp, model_title.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "def effb1_244_v4(weights='noisy-student'):\n",
    "    \n",
    "    effb1 = efn.EfficientNetB1(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    \n",
    "    model_image = Sequential([\n",
    "        effb1,\n",
    "        GlobalAveragePooling2D(name='effb1-pooling'),\n",
    "    ], name='effb1-img')\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(420, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.5, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([model_image.output, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[model_image.input, model_title.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "def effb3(weights='noisy-student'):\n",
    "    img_inp = Input(shape=(*IMAGE_SIZE, 3))\n",
    "    effb3 = efn.EfficientNetB3(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    \n",
    "    pt_depth = effb3.layers[-1].get_output_shape_at(0)[-1]\n",
    "    pt_features = effb3(img_inp)\n",
    "    bn_features = BatchNormalization()(pt_features)\n",
    "    \n",
    "    attn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(Dropout(0.5)(bn_features))\n",
    "    attn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(8, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(1, kernel_size = (1,1), padding = 'valid', activation = 'sigmoid')(attn_layer)\n",
    "    \n",
    "    up_c2_w = np.ones((1, 1, 1, pt_depth))\n",
    "    up_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', activation = 'linear', \n",
    "                   use_bias = False, weights = [up_c2_w])\n",
    "    up_c2.trainable = False\n",
    "    attn_layer = up_c2(attn_layer)\n",
    "    \n",
    "    mask_features = multiply([attn_layer, bn_features])\n",
    "    gap_features = GlobalAveragePooling2D()(mask_features)\n",
    "    gap_mask = GlobalAveragePooling2D()(attn_layer)\n",
    "    # to account for missing values from the attention model\n",
    "    gap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\n",
    "    gap_dr = Dropout(0.25)(gap)\n",
    "    img_embed = Dense(2400, activation = 'swish', name='img-embed')(gap_dr)\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([img_embed, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[img_inp, model_title.input, label], outputs=output)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "brief-professional",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:49.637730Z",
     "iopub.status.busy": "2021-06-15T03:16:49.636476Z",
     "iopub.status.idle": "2021-06-15T03:16:49.638744Z",
     "shell.execute_reply": "2021-06-15T03:16:49.639261Z",
     "shell.execute_reply.started": "2021-06-15T03:06:55.488359Z"
    },
    "papermill": {
     "duration": 0.04812,
     "end_time": "2021-06-15T03:16:49.639387",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.591267",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def incepV2(weights='imagenet'):\n",
    "    inceptionV2 = InceptionResNetV2(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    for layer in inceptionV2.layers:\n",
    "        layer.trainable = False\n",
    "        \n",
    "    model_image = Sequential([\n",
    "        inceptionV2,\n",
    "        GlobalAveragePooling2D(name='incep-pooling'),\n",
    "        BatchNormalization(name='incep_bn1'),\n",
    "        Dropout(0.2),\n",
    "        Dense(2400, name='incep_dense1'),\n",
    "        Activation('swish', name='incep_act1'),\n",
    "    ], name='incep-img')\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 70, input_length=100, name='title-embed'),\n",
    "#         GlobalAveragePooling1D(name='title-pooling'),\n",
    "        Flatten(name='title-flatten'),\n",
    "        BatchNormalization(name='title-bn1'),\n",
    "        Dropout(0.2),\n",
    "        Dense(650, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "#         Dropout(0.1),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    eff_aux = Model(\n",
    "        inputs = inceptionV2.input, \n",
    "        outputs = inceptionV2.get_layer('block17_7_mixed').output)\n",
    "    aux_model = Sequential([\n",
    "        eff_aux,\n",
    "        Conv2D(128, kernel_size=(3, 3), activation='relu', name='aux-conv1'),\n",
    "        BatchNormalization(name='aux-bn1'),\n",
    "        GlobalAveragePooling2D(name='aux-pooling'),\n",
    "        Dropout(0.5),\n",
    "        Dense(1024, name='aux-dense1'),\n",
    "        Activation('swish'),\n",
    "    ], name='aux-model')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.5, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([model_image.output, model_title.output])\n",
    "    concatenate2 = Concatenate(name='concatenate2')([concatenate, aux_model.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate2, label])\n",
    "#     embeddings = Dense(3050, activation='swish', name='embedding')(concatenate)\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[model_image.input, aux_model.input, model_title.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "def incepV2_512_v3(weights='imagenet'):\n",
    "    inceptionV2 = InceptionResNetV2(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    \n",
    "    model_image = Sequential([\n",
    "        inceptionV2,\n",
    "        GlobalAveragePooling2D(name='incep-pooling'),\n",
    "        BatchNormalization(name='incep_bn1'),\n",
    "        Dropout(0.2),\n",
    "        Dense(2400, name='incep_dense1'),\n",
    "        Activation('swish', name='incep_act1'),\n",
    "    ], name='incep-img')\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([model_image.output, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[model_image.input, model_title.input, label], outputs=output)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "driven-avatar",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:49.713931Z",
     "iopub.status.busy": "2021-06-15T03:16:49.712705Z",
     "iopub.status.idle": "2021-06-15T03:16:49.715409Z",
     "shell.execute_reply": "2021-06-15T03:16:49.714999Z",
     "shell.execute_reply.started": "2021-06-15T03:06:55.699919Z"
    },
    "papermill": {
     "duration": 0.04795,
     "end_time": "2021-06-15T03:16:49.715513",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.667563",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def xception(weights='imagenet'):\n",
    "    xcep = Xception(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    \n",
    "    for layer in xcep.layers:\n",
    "        layer.trainable = False\n",
    "        \n",
    "    model_image = Sequential([\n",
    "        xcep,\n",
    "        GlobalAveragePooling2D(name='incep-pooling'),\n",
    "        BatchNormalization(name='incep_bn1'),\n",
    "        Dropout(0.3),\n",
    "        Dense(2400, name='incep_dense1'),\n",
    "        Activation('swish', name='incep_act1'),\n",
    "    ], name='incep-img')\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 100, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    aux_model = Model(inputs=xcep.input, outputs=xcep.get_layer('block8_sepconv3_act').output)\n",
    "    aux_model = Sequential([\n",
    "        aux_model,\n",
    "        Conv2D(128, kernel_size=(3, 3), activation='relu', name='aux-conv1'),\n",
    "        BatchNormalization(name='aux-bn1'),\n",
    "        GlobalAveragePooling2D(name='aux-pooling'),\n",
    "        Dropout(0.5),\n",
    "        Dense(240, name='aux-dense1'),\n",
    "        Activation('swish'),\n",
    "    ])\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([model_image.output, model_title.output])\n",
    "    concatenate2 = Concatenate(name='concatenate2')([concatenate, aux_model.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate2, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[model_image.input, aux_model.input, model_title.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "def xception_512_v3(weights='imagenet'):\n",
    "    xcep = Xception(weights=weights, include_top=False, input_shape=(*IMAGE_SIZE, 3))\n",
    "    \n",
    "    model_image = Sequential([\n",
    "        xcep,\n",
    "        GlobalAveragePooling2D(name='incep-pooling'),\n",
    "        BatchNormalization(name='incep_bn1'),\n",
    "        Dropout(0.2),\n",
    "        Dense(2400, name='incep_dense1'),\n",
    "        Activation('swish', name='incep_act1'),\n",
    "    ], name='incep-img')\n",
    "        \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([model_image.output, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    \n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[model_image.input, model_title.input, label], outputs=output)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "editorial-slide",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:49.791641Z",
     "iopub.status.busy": "2021-06-15T03:16:49.778070Z",
     "iopub.status.idle": "2021-06-15T03:16:49.794796Z",
     "shell.execute_reply": "2021-06-15T03:16:49.794321Z",
     "shell.execute_reply.started": "2021-06-15T03:06:56.913770Z"
    },
    "papermill": {
     "duration": 0.051864,
     "end_time": "2021-06-15T03:16:49.794893",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.743029",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_nfnet_f0(weights='gs://kds-b135c9e4ed7b978eb1b05abe18f78523f1fe076527cd1f64555bd7b1/NFNET-F0/F0_NFNet'):\n",
    "    nfnet_ = NFNet(\n",
    "        num_classes=1000,\n",
    "        variant='F0',\n",
    "        drop_rate=0.2,\n",
    "        label_smoothing=0.1,\n",
    "        ema_decay=0.99999,\n",
    "        clipping_factor=0.01,\n",
    "        include_top=False,\n",
    "    )\n",
    "    if weights is not None:\n",
    "        nfnet_.load_weights(weights)\n",
    "\n",
    "    model_image = Sequential([\n",
    "        nfnet_,\n",
    "        GlobalAveragePooling2D(name='effb1-pooling'),\n",
    "        BatchNormalization(name='effb1_bn1'),\n",
    "        Dropout(0.2),\n",
    "        Dense(2400, name='effb1_dense1'),\n",
    "        Activation('swish', name='effb1_act1'),\n",
    "    ], name='effb1-img')\n",
    "    model_image.build((None, *IMAGE_SIZE, 3))\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1, name='title-conv'),\n",
    "        GlobalMaxPool1D(name='title-globalMax'),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([model_image.output, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[model_image.input, model_title.input, label], outputs=output)\n",
    "    return model\n",
    "\n",
    "def get_nfnet_f1(weights='gs://kds-1d6d6565dd2a34e7cc1a72d6d664184726bc52e3022d6010d9f9b173/NFNET-F1/F1_NFNet'):\n",
    "    \n",
    "    img_inp = Input(shape=(*IMAGE_SIZE, 3))\n",
    "    nfnet_ = NFNet(\n",
    "        num_classes=1000,\n",
    "        variant='F1',\n",
    "        drop_rate=0.2,\n",
    "        label_smoothing=0.1,\n",
    "        ema_decay=0.99999,\n",
    "        clipping_factor=0.01,\n",
    "        include_top=False,\n",
    "    )\n",
    "    if weights is not None:\n",
    "        nfnet_.load_weights(weights)\n",
    "    \n",
    "    pt_depth = 3072\n",
    "    pt_features = nfnet_(img_inp)\n",
    "    bn_features = BatchNormalization()(pt_features)\n",
    "    \n",
    "    attn_layer = Conv2D(64, kernel_size = (1,1), padding = 'same', activation = 'relu')(Dropout(0.5)(bn_features))\n",
    "    attn_layer = Conv2D(16, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(8, kernel_size = (1,1), padding = 'same', activation = 'relu')(attn_layer)\n",
    "    attn_layer = Conv2D(1, kernel_size = (1,1), padding = 'valid', activation = 'sigmoid')(attn_layer)\n",
    "    \n",
    "    up_c2_w = np.ones((1, 1, 1, pt_depth))\n",
    "    up_c2 = Conv2D(pt_depth, kernel_size = (1,1), padding = 'same', activation = 'linear', \n",
    "                   use_bias = False, weights = [up_c2_w])\n",
    "    up_c2.trainable = False\n",
    "    attn_layer = up_c2(attn_layer)\n",
    "    \n",
    "    mask_features = multiply([attn_layer, bn_features])\n",
    "    gap_features = GlobalAveragePooling2D()(mask_features)\n",
    "    gap_mask = GlobalAveragePooling2D()(attn_layer)\n",
    "    # to account for missing values from the attention model\n",
    "    gap = Lambda(lambda x: x[0]/x[1], name = 'RescaleGAP')([gap_features, gap_mask])\n",
    "    gap_dr = Dropout(0.25)(gap)\n",
    "    img_embed = Dense(2400, activation = 'swish', name='img-embed')(gap_dr)\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1, name='title-conv'),\n",
    "        GlobalMaxPool1D(name='title-globalMax'),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "\n",
    "    concatenate = Concatenate(name='concatenate')([img_embed, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[img_inp, model_title.input, label], outputs=output)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "great-window",
   "metadata": {
    "papermill": {
     "duration": 0.027616,
     "end_time": "2021-06-15T03:16:49.850548",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.822932",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Bert model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "caroline-joshua",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:49.929984Z",
     "iopub.status.busy": "2021-06-15T03:16:49.928715Z",
     "iopub.status.idle": "2021-06-15T03:16:49.931488Z",
     "shell.execute_reply": "2021-06-15T03:16:49.931067Z",
     "shell.execute_reply.started": "2021-06-15T03:06:58.876609Z"
    },
    "papermill": {
     "duration": 0.053109,
     "end_time": "2021-06-15T03:16:49.931586",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.878477",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "bert_model = '../input/bert-base-uncased-220421/bert_base'\n",
    "\n",
    "def get_bert_model(mname):\n",
    "    \n",
    "    idx = layers.Input((105), dtype=\"int32\", name=\"input_idx\")\n",
    "    masks = layers.Input((105), dtype=\"int32\", name=\"input_masks\")\n",
    "    \n",
    "    nlp = transformers.TFBertModel.from_pretrained(mname)\n",
    "    bert_out= nlp([idx, masks])[0]\n",
    "    \n",
    "    ## fine-tuning\n",
    "    x = layers.GlobalAveragePooling1D()(bert_out)\n",
    "    x = layers.Dense(750, activation=\"swish\", name='text-embed')(x)\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 100, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "    \n",
    "    concatenate = Concatenate(name='concatenate')([x, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    \n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "\n",
    "    # Compile model\n",
    "    model = tf.keras.Model(inputs=[idx, masks, model_title.input, label], outputs=[output])\n",
    "    return model\n",
    "\n",
    "xlm_model_base = '../input/tf-xlm-roberta-base'\n",
    "def xlm_roberta(mname):\n",
    "    \n",
    "    idx = layers.Input((105), dtype=\"int32\", name=\"input_idx\")\n",
    "    masks = layers.Input((105), dtype=\"int32\", name=\"input_masks\")\n",
    "    \n",
    "#     nlp = TFAutoModel.from_pretrained(mname)\n",
    "    nlp = transformers.TFXLMRobertaModel.from_pretrained(mname)\n",
    "    bert_out= nlp([idx, masks])[0]\n",
    "    \n",
    "    ## fine-tuning\n",
    "    x = layers.GlobalAveragePooling1D()(bert_out)\n",
    "    x = layers.Dense(750, activation=\"swish\", name='text-embed')(x)\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.5, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "    \n",
    "    concatenate = Concatenate(name='concatenate')([x, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[idx, masks, model_title.input, label], outputs=[output])\n",
    "    \n",
    "    return model\n",
    "\n",
    "roberta_base_id_model = '../input/tfroberta-base-indonesian/roberta-base-indonesian-522M'\n",
    "def get_roberta_base_id(mname = roberta_base_id_model):\n",
    "    \n",
    "    idx = layers.Input((105), dtype=\"int32\", name=\"input_idx\")\n",
    "    masks = layers.Input((105), dtype=\"int32\", name=\"input_masks\")\n",
    "    \n",
    "    nlp = TFAutoModel.from_pretrained(mname)\n",
    "    bert_out= nlp([idx, masks])[0]\n",
    "    \n",
    "    ## fine-tuning\n",
    "    x = layers.GlobalAveragePooling1D()(bert_out)\n",
    "    x = layers.Dense(750, activation=\"swish\", name='text-embed')(x)\n",
    "    \n",
    "    model_title = Sequential([\n",
    "        Input(shape=(100,), name='title-input'),\n",
    "        Embedding(25000, 150, input_length=100, name='title-embed'),\n",
    "        Dropout(0.2),\n",
    "        Conv1D(300, 3, padding='valid', activation='relu', strides=1),\n",
    "        GlobalMaxPool1D(),\n",
    "        Dense(720, name='title-dense1'), #650 -> 0.81\n",
    "        Activation('swish', name='title-act1'),\n",
    "        Dropout(0.2),\n",
    "        \n",
    "        Dense(650, name='title-dense2'),\n",
    "        BatchNormalization(name='title-bn2'),\n",
    "        Activation('swish', name='title-act2'),\n",
    "    ], name='title-vec')\n",
    "    \n",
    "    margin = ArcMarginProduct(\n",
    "        n_classes = CLASSES, \n",
    "        s = 30, \n",
    "        m = 0.7, \n",
    "        name='head/arc_margin', \n",
    "        dtype='float32'\n",
    "    )\n",
    "    \n",
    "    concatenate = Concatenate(name='concatenate')([x, model_title.output])\n",
    "    label = Input(shape=(), name='arc-input')\n",
    "    arc_face = margin([concatenate, label])\n",
    "    output = Dense(CLASSES, activation='softmax', name='output')(arc_face)\n",
    "    \n",
    "    model = Model(inputs=[idx, masks, model_title.input, label], outputs=[output])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "judicial-theater",
   "metadata": {
    "papermill": {
     "duration": 0.028348,
     "end_time": "2021-06-15T03:16:49.987724",
     "exception": false,
     "start_time": "2021-06-15T03:16:49.959376",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Help functions to get embeddings and predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "adult-namibia",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:50.058214Z",
     "iopub.status.busy": "2021-06-15T03:16:50.056855Z",
     "iopub.status.idle": "2021-06-15T03:16:50.059685Z",
     "shell.execute_reply": "2021-06-15T03:16:50.059261Z",
     "shell.execute_reply.started": "2021-06-15T03:07:00.970141Z"
    },
    "papermill": {
     "duration": 0.043883,
     "end_time": "2021-06-15T03:16:50.059784",
     "exception": false,
     "start_time": "2021-06-15T03:16:50.015901",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_image_embeddings(model):\n",
    "    preds = []\n",
    "    chunk = 5000\n",
    "    iterator = np.arange(np.ceil(len(df)/chunk))\n",
    "    for j in iterator:\n",
    "        a = int(j * chunk)\n",
    "        b = int((j+1) * chunk)\n",
    "        img_paths, input_ids, att_mask, title_vec = read_dataset(df.iloc[a:b])\n",
    "        image_dataset = get_dataset(img_paths, title_vec)\n",
    "        img_embeddings = model.predict(image_dataset)\n",
    "        preds.append(img_embeddings)\n",
    "    del model, img_paths, title_vec\n",
    "    img_embeddings = np.concatenate(preds)\n",
    "    del preds\n",
    "    return img_embeddings\n",
    "\n",
    "def get_text_embeddings(model):\n",
    "    preds = []\n",
    "    chunk = 5000\n",
    "    iterator = np.arange(np.ceil(len(df)/chunk))\n",
    "    for j in iterator:\n",
    "        a = int(j * chunk)\n",
    "        b = int((j+1) * chunk)\n",
    "        img_paths, input_ids, att_mask, title_vec = read_dataset(df.iloc[a:b])\n",
    "        image_dataset = get_text_dataset(input_ids, att_mask, title_vec)\n",
    "        img_embeddings = model.predict(image_dataset)\n",
    "        preds.append(img_embeddings)\n",
    "    del model, img_paths, title_vec\n",
    "    img_embeddings = np.concatenate(preds)\n",
    "    del preds\n",
    "    return img_embeddings\n",
    "\n",
    "def get_neighbors(embeddings, KNN=50):\n",
    "    KNN = 50 if len(embeddings)>3 else 3\n",
    "        \n",
    "    model = NearestNeighbors(n_neighbors=KNN, metric = 'correlation')\n",
    "    nearest_model = model.fit(embeddings)\n",
    "    distances, indices = nearest_model.kneighbors(embeddings)\n",
    "\n",
    "    return distances, indices\n",
    "\n",
    "def get_predictions(number_of_embeds, distances, indices, th=40):\n",
    "    # get predictions\n",
    "    predictions = []\n",
    "    for k in range(number_of_embeds):\n",
    "        idx = np.where(distances[k,] < th)[0]\n",
    "        ids = indices[k, idx]\n",
    "        posting_ids = np.unique(df['posting_id'].iloc[ids].values)\n",
    "        predictions.append(posting_ids)\n",
    "        \n",
    "    for th1 in np.arange(th, th+0.3, 0.02):\n",
    "        for k in range(number_of_embeds):\n",
    "            if len(predictions[k]) <= 1:\n",
    "                idx = np.where(distances[k,] < th1)[0]\n",
    "                ids = indices[k, idx]\n",
    "                posting_ids = np.unique(df['posting_id'].iloc[ids].values)\n",
    "                predictions[k] = np.concatenate([predictions[k], posting_ids])\n",
    "                predictions[k] = np.unique(predictions[k])\n",
    "            \n",
    "    return predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "billion-treasurer",
   "metadata": {
    "papermill": {
     "duration": 0.02795,
     "end_time": "2021-06-15T03:16:50.116524",
     "exception": false,
     "start_time": "2021-06-15T03:16:50.088574",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Get model choices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "steady-johnson",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:50.198059Z",
     "iopub.status.busy": "2021-06-15T03:16:50.190016Z",
     "iopub.status.idle": "2021-06-15T03:16:50.200123Z",
     "shell.execute_reply": "2021-06-15T03:16:50.200499Z",
     "shell.execute_reply.started": "2021-06-15T03:07:03.969262Z"
    },
    "papermill": {
     "duration": 0.056211,
     "end_time": "2021-06-15T03:16:50.200618",
     "exception": false,
     "start_time": "2021-06-15T03:16:50.144407",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_model(weight_path, model_name):\n",
    "    if model_name == 'effb1':\n",
    "        model = effb1(weights=None)\n",
    "        for layer in model.get_layer('efficientnet-b1').layers[-170:]:\n",
    "            if not isinstance(layer, layers.BatchNormalization):\n",
    "                layer.trainable = True\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[2]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'effb1_512_v2':\n",
    "        model = effb1_512_v2(weights=None)\n",
    "        for layer in model.get_layer('efficientnet-b1').layers[-170:]:\n",
    "            if not isinstance(layer, layers.BatchNormalization):\n",
    "                layer.trainable = True\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[2]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'effb1_512_v3':\n",
    "        model = effb1_512_v3(weights=None)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'effb1_244_v4':\n",
    "        model = effb1_244_v4(weights=None)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'effb2':\n",
    "        model = effb2(weights=None)\n",
    "        for layer in model.get_layer('efficientnet-b2').layers[-190:]:\n",
    "            if not isinstance(layer, layers.BatchNormalization):\n",
    "                layer.trainable = True\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0]], outputs=model.get_layer('effb1_act1').output)\n",
    "    elif model_name == 'effb3':\n",
    "        model = effb3(weights=None)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'effb5':\n",
    "        model = effb5(weights=None)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'effb5_v2':\n",
    "        model = effb5_v2(weights=None)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'effb7':\n",
    "        model = effb7(weights=None)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0]], outputs=model.get_layer('img-embed').output)\n",
    "    elif model_name == 'effb7_v2':\n",
    "        model = effb7_v2(weights=None)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'incepv2':\n",
    "        model = incepV2(weights=None)\n",
    "        for layer in model.get_layer('inception_resnet_v2').layers[-380:]:\n",
    "            if not isinstance(layer, layers.BatchNormalization):\n",
    "                layer.trainable = True\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[2]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'incepV2_512_v3':\n",
    "        model = incepV2_512_v3(weights=None)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'xception':\n",
    "        model = xception_512(weights=None)\n",
    "        for layer in model.get_layer('xception').layers[-80:]:\n",
    "            if not isinstance(layer, layers.BatchNormalization):\n",
    "                layer.trainable = True\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[2]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'xception_512_v3':\n",
    "        model = xception_512_v3(weights=None)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'nfnet_f0':\n",
    "        model = get_nfnet_f0(weights=None)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'nfnet_f1':\n",
    "        model = get_nfnet_f1(weights=None)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1]], outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'bert':\n",
    "        model = get_bert_model(bert_model)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1], model.input[2]], \n",
    "                      outputs=model.get_layer('concatenate').output)\n",
    "    elif model_name == 'xlm-roberta':\n",
    "        model = xlm_roberta(xlm_model_base)\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1], model.input[2]], \n",
    "                      outputs=model.get_layer('concatenate').output) \n",
    "    elif model_name == 'roberta_base_id':\n",
    "        model = get_roberta_base_id()\n",
    "        model.load_weights(weight_path)\n",
    "        model = Model(inputs=[model.input[0], model.input[1], model.input[2]], \n",
    "                      outputs=model.get_layer('concatenate').output) \n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "seasonal-coordinate",
   "metadata": {
    "papermill": {
     "duration": 0.027934,
     "end_time": "2021-06-15T03:16:50.256364",
     "exception": false,
     "start_time": "2021-06-15T03:16:50.228430",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Image predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "abandoned-youth",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:16:50.331436Z",
     "iopub.status.busy": "2021-06-15T03:16:50.330665Z",
     "iopub.status.idle": "2021-06-15T03:17:56.356391Z",
     "shell.execute_reply": "2021-06-15T03:17:56.356834Z",
     "shell.execute_reply.started": "2021-06-15T03:11:19.850494Z"
    },
    "papermill": {
     "duration": 66.072921,
     "end_time": "2021-06-15T03:17:56.356995",
     "exception": false,
     "start_time": "2021-06-15T03:16:50.284074",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get predictions of model 0\n",
      "Get predictions of model 1\n",
      "Get predictions of model 2\n",
      "Get predictions of model 3\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "77920"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_weights = [\n",
    "    ('../input/shopee-effb3-512/model_weights.h5', 'effb3', 0.35, (512, 512), False),\n",
    "    ('../input/shopee-effb5-512-v2/model_weights.h5', 'effb5_v2', 0.35, (512, 512), False),\n",
    "    ('../input/shopee-nfnet-512/model_weights.h5', 'nfnet_f0', 0.35, (512, 512), False),\n",
    "    ('../input/shopee-nfnet-f1-512/model_weights.h5', 'nfnet_f1', 0.35, (512, 512), False),\n",
    "]\n",
    "\n",
    "use_weight = False\n",
    "img_predictions = []\n",
    "img_embeds_avg = []\n",
    "for n, (weight_path, model_name, th, img_size, only_img) in enumerate(model_weights):\n",
    "    print(f'Get predictions of model {n}')\n",
    "    CLASSES = [11014, 11014, 11014, 11014, 11014][n] \n",
    "    IMAGE_SIZE = img_size\n",
    "    ONLY_IMAGE = only_img\n",
    "    model = get_model(weight_path, model_name)\n",
    "    # get image embeds\n",
    "    img_embed = get_image_embeddings(model)\n",
    "    del model\n",
    "\n",
    "    # scale embedding\n",
    "    scaler = StandardScaler()\n",
    "    img_embed = scaler.fit_transform(img_embed)\n",
    "\n",
    "    if use_weight:\n",
    "        weights = [1, 1, 1, 1, 1]\n",
    "        img_embeds_avg.append(img_embed)\n",
    "        del img_embed, scaler\n",
    "    else:\n",
    "        # get neighbors\n",
    "        distances, indices = get_neighbors(img_embed)\n",
    "\n",
    "        # get predictions\n",
    "        preds = get_predictions(img_embed.shape[0], distances, indices, th=th)\n",
    "        img_predictions.append(preds)\n",
    "\n",
    "        del distances, indices, img_embed, preds, scaler\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "drawn-laundry",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:17:56.632588Z",
     "iopub.status.busy": "2021-06-15T03:17:56.631519Z",
     "iopub.status.idle": "2021-06-15T03:17:56.634839Z",
     "shell.execute_reply": "2021-06-15T03:17:56.635257Z",
     "shell.execute_reply.started": "2021-06-15T03:12:29.613928Z"
    },
    "papermill": {
     "duration": 0.24944,
     "end_time": "2021-06-15T03:17:56.635405",
     "exception": false,
     "start_time": "2021-06-15T03:17:56.385965",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if use_weight:\n",
    "    img_embeds_avg = np.concatenate([\n",
    "        img_embeds_avg[0], \n",
    "        img_embeds_avg[1], \n",
    "        img_embeds_avg[2], \n",
    "        img_embeds_avg[3],\n",
    "    ], axis=1)\n",
    "    distances, indices = get_neighbors(img_embeds_avg)\n",
    "    preds = get_predictions(img_embeds_avg.shape[0], distances, indices, th=0.35)\n",
    "    img_predictions.append(preds)\n",
    "    del img_embeds_avg\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "respective-triumph",
   "metadata": {
    "papermill": {
     "duration": 0.029351,
     "end_time": "2021-06-15T03:17:56.694446",
     "exception": false,
     "start_time": "2021-06-15T03:17:56.665095",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Text predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "devoted-raising",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:17:56.770574Z",
     "iopub.status.busy": "2021-06-15T03:17:56.765524Z",
     "iopub.status.idle": "2021-06-15T03:18:24.016716Z",
     "shell.execute_reply": "2021-06-15T03:18:24.017128Z",
     "shell.execute_reply.started": "2021-06-15T03:14:01.904135Z"
    },
    "papermill": {
     "duration": 27.293805,
     "end_time": "2021-06-15T03:18:24.017302",
     "exception": false,
     "start_time": "2021-06-15T03:17:56.723497",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
      "/opt/conda/lib/python3.7/site-packages/transformers/tokenization_utils_base.py:2079: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
      "  FutureWarning,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get predictions of model 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "All model checkpoint layers were used when initializing TFRobertaModel.\n",
      "\n",
      "All the layers of TFRobertaModel were initialized from the model checkpoint at ../input/tfroberta-base-indonesian/roberta-base-indonesian-522M.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "41127"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_predictions = []\n",
    "bert_predictions = []\n",
    "txt_embeds_avg = []\n",
    "\n",
    "# for roberta base indonesian\n",
    "# extract vocab from train data\n",
    "df = pd.read_csv('../input/shopee-product-matching/train.csv')\n",
    "    \n",
    "df['title'] = df['title'].apply(lambda x: remove_concatenate_2_words(x))\n",
    "df['title'] = df['title'].str.lower()\n",
    "df['title'] = df['title'].apply(lambda x: remove_punctuation(x))\n",
    "df['title'] = df['title'].apply(lambda x: str(x).split())\n",
    "df['title'] = df['title'].apply(lambda x: remove_stopwords(x))\n",
    "df['title'] = df['title'].apply(lambda x: remove_zero_val(x))\n",
    "df['title'] = df['title'].apply(lambda x: remove_strange_words(x))\n",
    "df['title'] = df['title'].apply(lambda x: list(np.unique(x)))\n",
    "\n",
    "# title vocab\n",
    "words = list(df['title'])\n",
    "train_vocab = list(np.unique(np.concatenate(words)))\n",
    "\n",
    "df = pd.read_csv('../input/shopee-product-matching/test.csv')\n",
    "df['tmp'] = df['title'].apply(lambda x: string_escape(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_concatenate_2_words(x))\n",
    "df['tmp'] = df['tmp'].str.lower()\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_punctuation(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: str(x).split())\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_stopwords(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_zero_val(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: remove_strange_words(x))\n",
    "df['tmp'] = df['tmp'].apply(lambda x: list(np.unique(x)))\n",
    "\n",
    "# for mlp input\n",
    "# title vocab\n",
    "words = list(df['tmp'])\n",
    "words = list(np.unique(np.concatenate(words)))\n",
    "words = train_vocab + words\n",
    "\n",
    "# Text vectorizer\n",
    "model = text_vectorizer(max_features = 25000, max_len = 100, vocab = words)\n",
    "list_text = [' '.join(x) for x in df['tmp']]\n",
    "title_vec = model.predict(list_text)\n",
    "df['title_vec'] = list(title_vec)\n",
    "del model, list_text, title_vec, words, train_vocab\n",
    "\n",
    "MAX_LEN = 105\n",
    "MODEL = '../input/tfroberta-base-indonesian/roberta-base-indonesian-522M'\n",
    "tokenizer = transformers.AutoTokenizer.from_pretrained(MODEL)\n",
    "\n",
    "df['tmp'] = df['title'].apply(lambda x: string_escape(x))\n",
    "df[\"tmp\"] = df[\"tmp\"].apply(lambda x: utils_preprocess_text(\n",
    "    x, flg_stemm=False, flg_lemm=False, lst_stopwords=None))\n",
    "\n",
    "# for BERT\n",
    "ids, att_mask = regular_encode(list(df[\"tmp\"].values), tokenizer, maxlen=MAX_LEN)\n",
    "df['input_ids'] = list(ids)\n",
    "df['att_mask'] = list(att_mask)\n",
    "del ids, att_mask\n",
    "\n",
    "# bad, decrease the LB\n",
    "model_weights = [\n",
    "    ('../input/shopee-roberta-base-id/model_weights.h5', 'roberta_base_id', 0.50)\n",
    "]\n",
    "\n",
    "for n, (weight_path, model_name, th) in enumerate(model_weights):\n",
    "    print(f'Get predictions of model {n}')\n",
    "    CLASSES = [11014, 11014, 11014][n] \n",
    "    model = get_model(weight_path, model_name)\n",
    "    # get text embeds\n",
    "    txt_embed = get_text_embeddings(model)\n",
    "    del model\n",
    "\n",
    "    # scale embedding\n",
    "    scaler = StandardScaler()\n",
    "    txt_embed = scaler.fit_transform(txt_embed)\n",
    "    txt_embeds_avg.append(txt_embed)\n",
    "    \n",
    "    del txt_embed, scaler\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "advanced-computer",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:18:24.086201Z",
     "iopub.status.busy": "2021-06-15T03:18:24.085369Z",
     "iopub.status.idle": "2021-06-15T03:18:24.460541Z",
     "shell.execute_reply": "2021-06-15T03:18:24.460962Z",
     "shell.execute_reply.started": "2021-06-15T03:14:24.727500Z"
    },
    "papermill": {
     "duration": 0.41172,
     "end_time": "2021-06-15T03:18:24.461125",
     "exception": false,
     "start_time": "2021-06-15T03:18:24.049405",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "txt_embeds_avg = np.concatenate([\n",
    "    txt_embeds_avg[0], \n",
    "], axis=1)\n",
    "distances, indices = get_neighbors(txt_embeds_avg)\n",
    "preds = get_predictions(txt_embeds_avg.shape[0], distances, indices, th=0.50)\n",
    "text_predictions.append(preds)\n",
    "del txt_embeds_avg\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "smart-seminar",
   "metadata": {
    "papermill": {
     "duration": 0.032198,
     "end_time": "2021-06-15T03:18:24.528029",
     "exception": false,
     "start_time": "2021-06-15T03:18:24.495831",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### TFIDF predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "final-favor",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:18:24.613666Z",
     "iopub.status.busy": "2021-06-15T03:18:24.612809Z",
     "iopub.status.idle": "2021-06-15T03:18:40.795579Z",
     "shell.execute_reply": "2021-06-15T03:18:40.795982Z",
     "shell.execute_reply.started": "2021-06-15T03:14:30.121360Z"
    },
    "papermill": {
     "duration": 16.235855,
     "end_time": "2021-06-15T03:18:40.796193",
     "exception": false,
     "start_time": "2021-06-15T03:18:24.560338",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finding similar titles...\n",
      "chunk 0 to 3\n",
      "Finding similar titles...\n",
      "chunk 0 to 3\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_text_predictions(df, title_cu, max_features = 25_000):\n",
    "    \n",
    "    text_preds = []\n",
    "    for stopw in ['english', STOPWORDS_ID]:\n",
    "        model = TfidfVectorizer(stop_words = stopw, binary = True, max_features = max_features)\n",
    "        text_embeddings = model.fit_transform(title_cu).toarray()\n",
    "\n",
    "        preds = []\n",
    "        CHUNK = 1024*4\n",
    "\n",
    "        print('Finding similar titles...')\n",
    "        CTS = len(df)//CHUNK\n",
    "        if len(df)%CHUNK!=0: CTS += 1\n",
    "        for j in range( CTS ):\n",
    "\n",
    "            a = j*CHUNK\n",
    "            b = (j+1)*CHUNK\n",
    "            b = min(b,len(df))\n",
    "            print('chunk',a,'to',b)\n",
    "\n",
    "            # COSINE SIMILARITY DISTANCE\n",
    "            cts = cupy.matmul( text_embeddings, text_embeddings[a:b].T).T\n",
    "\n",
    "            for k in range(b-a):\n",
    "                IDX = cupy.where(cts[k,] > 0.75)[0]\n",
    "                o = df.iloc[cupy.asnumpy(IDX)].posting_id.values\n",
    "                preds.append(o)\n",
    "                \n",
    "        text_preds.append(preds)\n",
    "        del model,text_embeddings, preds\n",
    "        gc.collect()\n",
    "    return text_preds\n",
    "\n",
    "def string_escape(s, encoding='utf-8'):\n",
    "    return (\n",
    "        s.encode('latin1')  # To bytes, required by 'unicode-escape'\n",
    "        .decode('unicode-escape')  # Perform the actual octal-escaping decode\n",
    "        .encode('latin1')  # 1:1 mapping back to bytes\n",
    "        .decode(encoding)\n",
    "    )  # Decode original encoding\n",
    "\n",
    "df = pd.read_csv('../input/shopee-product-matching/test.csv')\n",
    "df['title'] = df['title'].apply(lambda x: string_escape(x))\n",
    "df['title'] = df['title'].apply(lambda x: remove_punctuation(x))\n",
    "\n",
    "title_cu = cudf.Series(df['title'])\n",
    "text_predictions = text_predictions + (get_text_predictions(df, title_cu, max_features = 25000))\n",
    "del title_cu\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "sporting-output",
   "metadata": {
    "papermill": {
     "duration": 0.031585,
     "end_time": "2021-06-15T03:18:40.860380",
     "exception": false,
     "start_time": "2021-06-15T03:18:40.828795",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Combine predictions (Image + BertText + TFIDF)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "macro-guyana",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:18:40.933282Z",
     "iopub.status.busy": "2021-06-15T03:18:40.932672Z",
     "iopub.status.idle": "2021-06-15T03:18:40.935817Z",
     "shell.execute_reply": "2021-06-15T03:18:40.936202Z",
     "shell.execute_reply.started": "2021-06-15T03:14:50.078784Z"
    },
    "papermill": {
     "duration": 0.044169,
     "end_time": "2021-06-15T03:18:40.936332",
     "exception": false,
     "start_time": "2021-06-15T03:18:40.892163",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def combine_predictions(row):\n",
    "    all_preds = []\n",
    "    res = []\n",
    "    if len(img_predictions)>0:\n",
    "        for i in range(len(img_predictions)):\n",
    "            all_preds.append(row[f'img_prediction_{i}'])\n",
    "        x = np.concatenate(all_preds)\n",
    "\n",
    "        # remove item if item count less than 1\n",
    "        c = Counter(x)\n",
    "        res = np.array([i for i in x if c[i] >= 3])\n",
    "        del x\n",
    "    \n",
    "    # text preds\n",
    "    txt_pred=[]\n",
    "    if len(text_predictions) > 0:\n",
    "        all_preds = []\n",
    "        for i in range(len(text_predictions)):\n",
    "            all_preds.append(row[f'text_prediction_{i}'])\n",
    "        x = np.concatenate(all_preds)\n",
    "\n",
    "        # remove item if item count less than 1\n",
    "        c = Counter(x)\n",
    "        txt_pred = np.array([i for i in x if c[i] >= 2])\n",
    "        del x, c\n",
    "    y=[]\n",
    "    if len(bert_predictions) > 0:\n",
    "        all_preds = []\n",
    "        for i in range(len(bert_predictions)):\n",
    "            all_preds.append(row[f'bert_prediction_{i}'])\n",
    "        y = np.concatenate(all_preds)\n",
    "    del all_preds\n",
    "    res = [res, txt_pred, y]\n",
    "    res = np.concatenate(res)\n",
    "    del txt_pred, y\n",
    "    \n",
    "    if GET_CV:\n",
    "        return np.unique(res)\n",
    "    else:\n",
    "        return ' '.join( np.unique(res) )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "threatened-soccer",
   "metadata": {
    "papermill": {
     "duration": 0.032031,
     "end_time": "2021-06-15T03:18:41.000058",
     "exception": false,
     "start_time": "2021-06-15T03:18:40.968027",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### Make submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "domestic-mailing",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:18:41.073180Z",
     "iopub.status.busy": "2021-06-15T03:18:41.072474Z",
     "iopub.status.idle": "2021-06-15T03:18:41.086475Z",
     "shell.execute_reply": "2021-06-15T03:18:41.085989Z",
     "shell.execute_reply.started": "2021-06-15T03:14:51.980971Z"
    },
    "papermill": {
     "duration": 0.054694,
     "end_time": "2021-06-15T03:18:41.086584",
     "exception": false,
     "start_time": "2021-06-15T03:18:41.031890",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for n, prediction in enumerate(img_predictions):\n",
    "    df[f'img_prediction_{n}'] = prediction\n",
    "for n, prediction in enumerate(bert_predictions):\n",
    "    df[f'bert_prediction_{n}'] = prediction\n",
    "for n, prediction in enumerate(text_predictions):\n",
    "    df[f'text_prediction_{n}'] = prediction\n",
    "df['matches'] = df.apply(combine_predictions, axis=1)\n",
    "df = df[['posting_id', 'matches']]\n",
    "df.to_csv('./submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "swedish-passage",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-06-15T03:18:41.162743Z",
     "iopub.status.busy": "2021-06-15T03:18:41.161926Z",
     "iopub.status.idle": "2021-06-15T03:18:41.166529Z",
     "shell.execute_reply": "2021-06-15T03:18:41.166092Z",
     "shell.execute_reply.started": "2021-06-15T03:14:53.913828Z"
    },
    "papermill": {
     "duration": 0.046777,
     "end_time": "2021-06-15T03:18:41.166633",
     "exception": false,
     "start_time": "2021-06-15T03:18:41.119856",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>posting_id</th>\n",
       "      <th>matches</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>test_2255846744</td>\n",
       "      <td>test_2255846744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>test_3588702337</td>\n",
       "      <td>test_3588702337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>test_4015706929</td>\n",
       "      <td>test_4015706929</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        posting_id          matches\n",
       "0  test_2255846744  test_2255846744\n",
       "1  test_3588702337  test_3588702337\n",
       "2  test_4015706929  test_4015706929"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('./submission.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "derived-smell",
   "metadata": {
    "papermill": {
     "duration": 0.033184,
     "end_time": "2021-06-15T03:18:41.232598",
     "exception": false,
     "start_time": "2021-06-15T03:18:41.199414",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 205.836327,
   "end_time": "2021-06-15T03:18:44.284429",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-06-15T03:15:18.448102",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}