Stage 3. Generate the predictions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sample_submission_stage_1.csv', 'sample_submission_stage_2.csv', 'test_stage_1.tsv', 'trees']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"data\"))\n",
    "import zipfile\n",
    "import sys\n",
    "import time\n",
    "\n",
    "from keras import backend, models, layers, initializers, regularizers, constraints, optimizers\n",
    "from keras import callbacks as kc\n",
    "from keras import optimizers as ko\n",
    "\n",
    "from sklearn.model_selection import cross_val_score, KFold, train_test_split\n",
    "from sklearn.metrics import log_loss\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate BERT embeddings for test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sample_submission_stage_1.csv', 'sample_submission_stage_2.csv', 'test_stage_1.tsv', 'trees']\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"data\"))\n",
    "import zipfile\n",
    "import sys\n",
    "import time\n",
    "\n",
    "import modeling\n",
    "import extract_features\n",
    "import tokenization\n",
    "import tensorflow as tf\n",
    "\n",
    "def compute_offset_no_spaces(text, offset):\n",
    "    count = 0\n",
    "    for pos in range(offset):\n",
    "        if text[pos] != \" \": count +=1\n",
    "    return count\n",
    "\n",
    "def count_chars_no_special(text):\n",
    "    count = 0\n",
    "    special_char_list = [\"#\"]\n",
    "    for pos in range(len(text)):\n",
    "        if text[pos] not in special_char_list: count +=1\n",
    "    return count\n",
    "\n",
    "def count_length_no_special(text):\n",
    "    count = 0\n",
    "    special_char_list = [\"#\", \" \"]\n",
    "    for pos in range(len(text)):\n",
    "        if text[pos] not in special_char_list: count +=1\n",
    "    return count\n",
    "\n",
    "def run_bert(data, output, os_command, embedding_size=1024, layer=-1):\n",
    "    '''\n",
    "    Runs a forward propagation of BERT on input text, extracting contextual word embeddings\n",
    "    Input: data, a pandas DataFrame containing the information in one of the GAP files\n",
    "\n",
    "    Output: emb, a pandas DataFrame containing contextual embeddings for the words A, B and Pronoun. Each embedding is a numpy array of shape (768)\n",
    "    columns: \"emb_A\": the embedding for word A\n",
    "             \"emb_B\": the embedding for word B\n",
    "             \"emb_P\": the embedding for the pronoun\n",
    "             \"label\": the answer to the coreference problem: \"A\", \"B\" or \"NEITHER\"\n",
    "    '''\n",
    "    # From the current file, take the text only, and write it in a file which will be passed to BERT\n",
    "    text = data[\"Text\"]\n",
    "    text.to_csv(\"input.txt\", index = False, header = False)\n",
    "\n",
    "    # The script extract_features.py runs forward propagation through BERT, and writes the output in the file output.jsonl\n",
    "    # I'm lazy, so I'm only saving the output of the last layer. Feel free to change --layers = -1 to save the output of other layers.\n",
    "\n",
    "    res = os.system(os_command)\n",
    "    bert_output = pd.read_json(\"output.jsonl\", lines = True)\n",
    "\n",
    "    os.system(\"rm output.jsonl\")\n",
    "    os.system(\"rm input.txt\")\n",
    "\n",
    "    index = data.index\n",
    "    columns = [\"emb_A\", \"emb_B\", \"emb_P\", \"label\"]\n",
    "    emb = pd.DataFrame(index = index, columns = columns)\n",
    "    emb.index.name = \"ID\"\n",
    "\n",
    "    for i in range(len(data)): # For each line in the data file\n",
    "        # get the words A, B, Pronoun. Convert them to lower case, since we're using the uncased version of BERT\n",
    "        P = data.loc[i,\"Pronoun\"].lower()\n",
    "        A = data.loc[i,\"A\"].lower()\n",
    "        B = data.loc[i,\"B\"].lower()\n",
    "\n",
    "        # For each word, find the offset not counting spaces. This is necessary for comparison with the output of BERT\n",
    "        P_offset = compute_offset_no_spaces(data.loc[i,\"Text\"], data.loc[i,\"Pronoun-offset\"])\n",
    "        A_offset = compute_offset_no_spaces(data.loc[i,\"Text\"], data.loc[i,\"A-offset\"])\n",
    "        B_offset = compute_offset_no_spaces(data.loc[i,\"Text\"], data.loc[i,\"B-offset\"])\n",
    "        # Figure out the length of A, B, not counting spaces or special characters\n",
    "        A_length = count_length_no_special(A)\n",
    "        B_length = count_length_no_special(B)\n",
    "\n",
    "        # Initialize embeddings with zeros\n",
    "        emb_A = np.zeros(embedding_size)\n",
    "        emb_B = np.zeros(embedding_size)\n",
    "        emb_P = np.zeros(embedding_size)\n",
    "\n",
    "        # Initialize counts\n",
    "        count_chars = 0\n",
    "        cnt_A, cnt_B, cnt_P = 0, 0, 0\n",
    "\n",
    "        features = pd.DataFrame(bert_output.loc[i,\"features\"]) # Get the BERT embeddings for the current line in the data file\n",
    "        for j in range(2,len(features)):  # Iterate over the BERT tokens for the current line; we skip over the first 2 tokens, which don't correspond to words\n",
    "            token = features.loc[j,\"token\"]\n",
    "\n",
    "            # See if the character count until the current token matches the offset of any of the 3 target words\n",
    "            if count_chars  == P_offset: \n",
    "                # print(token)\n",
    "                emb_P += np.array(features.loc[j,\"layers\"][0]['values'])\n",
    "                cnt_P += 1\n",
    "            if count_chars in range(A_offset, A_offset + A_length): \n",
    "                # print(token)\n",
    "                emb_A += np.array(features.loc[j,\"layers\"][0]['values'])\n",
    "                cnt_A +=1\n",
    "            if count_chars in range(B_offset, B_offset + B_length): \n",
    "                # print(token)\n",
    "                emb_B += np.array(features.loc[j,\"layers\"][0]['values'])\n",
    "                cnt_B +=1\t\t\t\t\t\t\t\t\n",
    "            # Update the character count\n",
    "            count_chars += count_length_no_special(token)\n",
    "        # Taking the average between tokens in the span of A or B, so divide the current value by the count\t\n",
    "        emb_A /= cnt_A\n",
    "        emb_B /= cnt_B\n",
    "\n",
    "        # Work out the label of the current piece of text\n",
    "        label = \"Neither\"\n",
    "        if (data.loc[i,\"A-coref\"] == True):\n",
    "            label = \"A\"\n",
    "        if (data.loc[i,\"B-coref\"] == True):\n",
    "            label = \"B\"\n",
    "\n",
    "        # Put everything together in emb\n",
    "        emb.iloc[i] = [emb_A, emb_B, emb_P, label]\n",
    "    return emb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stage_2_dataset_name = 'test_stage_2.tsv' # remember to modify this from 'test_stage_1.tsv' -> 'test_stage_2.tsv', or the code cannot run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_data = pd.read_csv(stage_2_dataset_name, sep='\\t')\n",
    "test_data['A-coref'] = False\n",
    "test_data['B-coref'] = False\n",
    "test_data['ID'] = 'development' + test_data['ID']\n",
    "test_data[['ID', 'Text', 'Pronoun', 'Pronoun-offset', 'A', 'A-offset', 'A-coref', \n",
    "          'B', 'B-offset', 'B-coref', 'URL']].to_csv(\"fixed_test_stage_2.tsv\", index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bert uncased base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:114: RuntimeWarning: invalid value encountered in true_divide\n",
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:115: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    }
   ],
   "source": [
    "tag = 'bert-base-uncased-seq512-'\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "tag = tag + str(layer)\n",
    "os_command = \"python extract_features.py \\\n",
    "      --input_file=input.txt \\\n",
    "      --output_file=output.jsonl \\\n",
    "      --vocab_file=uncased_L-12_H-768_A-12/vocab.txt \\\n",
    "      --bert_config_file=uncased_L-12_H-768_A-12/bert_config.json \\\n",
    "      --init_checkpoint=uncased_L-12_H-768_A-12/bert_model.ckpt \\\n",
    "      --layers={} \\\n",
    "      --max_seq_length=512 \\\n",
    "      --batch_size=1\".format(layer)\n",
    "\n",
    "test_data = pd.read_csv(\"fixed_test_stage_2.tsv\", sep = '\\t')\n",
    "test_emb = run_bert(test_data, embedding_size=embedding_size, os_command=os_command,\n",
    "                    output=\"{}contextual_embeddings_gap_test.json\".format(tag), layer=layer)\n",
    "test_emb.to_json(\"vector/bert_base/{}contextual_embeddings_gap_second_stage_test.json\".format(tag), orient = 'columns')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bert cased base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:114: RuntimeWarning: invalid value encountered in true_divide\n",
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:115: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    }
   ],
   "source": [
    "tag = 'bert-base-cased-seq512-'\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "tag = tag + str(layer)\n",
    "os_command = \"python extract_features.py \\\n",
    "      --input_file=input.txt \\\n",
    "      --output_file=output.jsonl \\\n",
    "      --vocab_file=cased_L-12_H-768_A-12/vocab.txt \\\n",
    "      --bert_config_file=cased_L-12_H-768_A-12/bert_config.json \\\n",
    "      --init_checkpoint=cased_L-12_H-768_A-12/bert_model.ckpt \\\n",
    "      --layers={} \\\n",
    "      --max_seq_length=512 \\\n",
    "      --batch_size=1\".format(layer)\n",
    "\n",
    "test_data = pd.read_csv(\"fixed_test_stage_2.tsv\", sep = '\\t')\n",
    "test_emb = run_bert(test_data, embedding_size=embedding_size, os_command=os_command,\n",
    "                    output=\"{}contextual_embeddings_gap_test.json\".format(tag), layer=layer)\n",
    "test_emb.to_json(\"vector/bert_base_cased/{}contextual_embeddings_gap_second_stage_test.json\".format(tag), orient = 'columns')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bert uncased large"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:114: RuntimeWarning: invalid value encountered in true_divide\n",
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:115: RuntimeWarning: invalid value encountered in true_divide\n"
     ]
    }
   ],
   "source": [
    "tag = 'bert-large-uncased-seq300-'\n",
    "embedding_size = 1024\n",
    "layer = 19\n",
    "tag = tag + str(layer)\n",
    "os_command = \"python extract_features.py \\\n",
    "      --input_file=input.txt \\\n",
    "      --output_file=output.jsonl \\\n",
    "      --vocab_file=uncased_L-24_H-1024_A-16/vocab.txt \\\n",
    "      --bert_config_file=uncased_L-24_H-1024_A-16/bert_config.json \\\n",
    "      --init_checkpoint=uncased_L-24_H-1024_A-16/bert_model.ckpt \\\n",
    "      --layers={} \\\n",
    "      --max_seq_length=300 \\\n",
    "      --batch_size=1\".format(layer)\n",
    "\n",
    "test_data = pd.read_csv(\"fixed_test_stage_2.tsv\", sep = '\\t')\n",
    "test_emb = run_bert(test_data, embedding_size=embedding_size, os_command=os_command,\n",
    "                    output=\"{}contextual_embeddings_gap_test.json\".format(tag), layer=layer)\n",
    "test_emb.to_json(\"vector/bert_big/{}contextual_embeddings_gap_second_stage_test.json\".format(tag), orient = 'columns')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bert cased large"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tag = 'bert-large-cased-seq300-'\n",
    "embedding_size = 1024\n",
    "layer = 18\n",
    "tag = tag + str(layer)\n",
    "os_command = \"python extract_features.py \\\n",
    "      --input_file=input.txt \\\n",
    "      --output_file=output.jsonl \\\n",
    "      --vocab_file=cased_L-24_H-1024_A-16/vocab.txt \\\n",
    "      --bert_config_file=cased_L-24_H-1024_A-16/bert_config.json \\\n",
    "      --init_checkpoint=cased_L-24_H-1024_A-16/bert_model.ckpt \\\n",
    "      --layers={} \\\n",
    "      --max_seq_length=300 \\\n",
    "      --batch_size=1\".format(layer)\n",
    "\n",
    "test_data = pd.read_csv(\"fixed_test_stage_2.tsv\", sep = '\\t')\n",
    "test_emb = run_bert(test_data, embedding_size=embedding_size, os_command=os_command,\n",
    "                    output=\"{}contextual_embeddings_gap_test.json\".format(tag), layer=layer)\n",
    "test_emb.to_json(\"vector/bert_big_cased/{}contextual_embeddings_gap_second_stage_test.json\".format(tag), orient = 'columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parse_json(embeddings, embedding_size):\n",
    "    '''\n",
    "    Parses the embeddigns given by BERT, and suitably formats them to be passed to the MLP model\n",
    "\n",
    "    Input: embeddings, a DataFrame containing contextual embeddings from BERT, as well as the labels for the classification problem\n",
    "    columns: \"emb_A\": contextual embedding for the word A\n",
    "             \"emb_B\": contextual embedding for the word B\n",
    "             \"emb_P\": contextual embedding for the pronoun\n",
    "             \"label\": the answer to the coreference problem: \"A\", \"B\" or \"NEITHER\"\n",
    "\n",
    "    Output: X, a numpy array containing, for each line in the GAP file, the concatenation of the embeddings of the target words\n",
    "            Y, a numpy array containing, for each line in the GAP file, the one-hot encoded answer to the coreference problem\n",
    "    '''\n",
    "    embeddings.sort_index(inplace = True) # Sorting the DataFrame, because reading from the json file messed with the order\n",
    "    X = np.zeros((len(embeddings),3* embedding_size))\n",
    "    Y = np.zeros((len(embeddings), 3))\n",
    "\n",
    "    # Concatenate features\n",
    "    for i in range(len(embeddings)):\n",
    "        A = np.array(embeddings.loc[i,\"emb_A\"])\n",
    "        B = np.array(embeddings.loc[i,\"emb_B\"])\n",
    "        P = np.array(embeddings.loc[i,\"emb_P\"])\n",
    "        X[i] = np.concatenate((A,B,P))\n",
    "\n",
    "    # One-hot encoding for labels\n",
    "    for i in range(len(embeddings)):\n",
    "        label = embeddings.loc[i,\"label\"]\n",
    "        if label == \"A\":\n",
    "            Y[i,0] = 1\n",
    "        elif label == \"B\":\n",
    "            Y[i,1] = 1\n",
    "        else:\n",
    "            Y[i,2] = 1\n",
    "\n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predictons for stage 1.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dev_filename = \"contextual_embeddings_gap_second_stage_test.json\"\n",
    "val_filename = \"contextual_embeddings_gap_validation.json\"\n",
    "test_filename = \"contextual_embeddings_gap_test.json\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dense_layer_sizes = [37]\n",
    "dropout_rate = 0.6\n",
    "learning_rate = 0.001\n",
    "n_fold = 5\n",
    "batch_size = 32\n",
    "epochs = 1000\n",
    "patience = 100\n",
    "# n_test = 100\n",
    "lambd = 0.1 # L2 regularization\n",
    "\n",
    "def build_mlp_model(input_shape):\n",
    "\tX_input = layers.Input(input_shape)\n",
    "\tX = layers.Dropout(dropout_rate, seed = 7)(X_input)\n",
    "\t# First dense layer\n",
    "\tX = layers.Dense(dense_layer_sizes[0], name = 'dense0')(X)\n",
    "\tX = layers.BatchNormalization(name = 'bn0')(X)\n",
    "\tX = layers.Activation('relu')(X)\n",
    "\tX = layers.Dropout(dropout_rate, seed = 7)(X)\n",
    "\n",
    "\t# Output layer\n",
    "\tX = layers.Dense(3, name = 'output', kernel_regularizer = regularizers.l2(lambd))(X)\n",
    "\tX = layers.Activation('softmax')(X)\n",
    "\n",
    "\t# Create model\n",
    "\tmodel = models.Model(input = X_input, output = X, name = \"classif_model\")\n",
    "\treturn model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_tag,\n",
    "                dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "                oof_folder='oof/', pred_folder='test_outputs/'):\n",
    "\n",
    "    tag = tag + str(layer) # follow the original naming style\n",
    "    dev_filename = tag + dev_filename\n",
    "    val_filename = tag + val_filename\n",
    "    test_filename = tag + test_filename\n",
    "    \n",
    "    development = pd.read_json(os.path.join(dev_folder_path, dev_filename))\n",
    "    X_development, Y_development = parse_json(development, embedding_size)\n",
    "\n",
    "    # We want predictions for all development rows. So instead of removing rows, make them 0\n",
    "    remove_development = [row for row in range(len(X_development)) if np.sum(np.isnan(X_development[row].reshape(-1)))]\n",
    "    X_development[remove_development] = np.zeros((3 * embedding_size))\n",
    "\n",
    "    prediction = np.zeros((len(X_development), 3)) # testing predictions\n",
    "\n",
    "    for fold_n in range(n_fold):\n",
    "        # split training and validation data\n",
    "        print('Make predictions for', fold_n, 'started at', time.ctime())\n",
    "        classif_model = build_mlp_model([X_development.shape[-1]])\n",
    "        classif_model.load_weights(os.path.join(checkpoint_path, model_tag + tag + str(fold_n) + '.pt'))\n",
    "\n",
    "        pred = classif_model.predict(x=X_development, verbose=0)\n",
    "        prediction += pred\n",
    "    \n",
    "    prediction /= n_fold\n",
    "    # prediction[remove_development] = [0.4, 0.4, 0.2] This is useless for prediction stage\n",
    "\n",
    "    submission = pd.read_csv(\"data/sample_submission_stage_2.csv\", index_col = \"ID\")\n",
    "    submission[\"A\"] = prediction[:,0]\n",
    "    submission[\"B\"] = prediction[:,1]\n",
    "    submission[\"NEITHER\"] = prediction[:,2]\n",
    "    submission.to_csv(pred_folder + model_tag + tag + \".csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 21:52:58 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:25: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 21:52:59 2019\n",
      "Make predictions for 2 started at Fri Apr 19 21:53:00 2019\n",
      "Make predictions for 3 started at Fri Apr 19 21:53:00 2019\n",
      "Make predictions for 4 started at Fri Apr 19 21:53:01 2019\n",
      "Make predictions for 5 started at Fri Apr 19 21:53:02 2019\n",
      "Make predictions for 6 started at Fri Apr 19 21:53:03 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_base\"\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-base-uncased-seq512-\"\n",
    "pred_tag = \"naive-\"\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 21:53:10 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:25: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 21:53:11 2019\n",
      "Make predictions for 2 started at Fri Apr 19 21:53:12 2019\n",
      "Make predictions for 3 started at Fri Apr 19 21:53:13 2019\n",
      "Make predictions for 4 started at Fri Apr 19 21:53:14 2019\n",
      "Make predictions for 5 started at Fri Apr 19 21:53:15 2019\n",
      "Make predictions for 6 started at Fri Apr 19 21:53:16 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_base_cased\"\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-base-cased-seq512-\"\n",
    "pred_tag = \"naive-\"\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 21:53:21 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:25: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 21:53:22 2019\n",
      "Make predictions for 2 started at Fri Apr 19 21:53:23 2019\n",
      "Make predictions for 3 started at Fri Apr 19 21:53:24 2019\n",
      "Make predictions for 4 started at Fri Apr 19 21:53:25 2019\n",
      "Make predictions for 5 started at Fri Apr 19 21:53:26 2019\n",
      "Make predictions for 6 started at Fri Apr 19 21:53:28 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_big\"\n",
    "embedding_size = 1024\n",
    "layer = 19\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-large-uncased-seq300-\"\n",
    "pred_tag = \"naive-\"\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 21:53:34 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:25: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 21:53:35 2019\n",
      "Make predictions for 2 started at Fri Apr 19 21:53:36 2019\n",
      "Make predictions for 3 started at Fri Apr 19 21:53:37 2019\n",
      "Make predictions for 4 started at Fri Apr 19 21:53:38 2019\n",
      "Make predictions for 5 started at Fri Apr 19 21:53:39 2019\n",
      "Make predictions for 6 started at Fri Apr 19 21:53:40 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_big_cased\"\n",
    "embedding_size = 1024\n",
    "layer = 18\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-large-cased-seq300-\"\n",
    "pred_tag = \"naive-\"\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predictons for stage 1.3\n",
    "\n",
    "### Model Architectures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_mlp_model_for_base_bert(input_shape, split_size):\n",
    "    X_input = layers.Input(input_shape)\n",
    "\n",
    "    # First dense layer\n",
    "    A = layers.Lambda(lambda x: x[:, :split_size])(X_input)\n",
    "    B = layers.Lambda(lambda x: x[:, split_size:split_size*2])(X_input)\n",
    "    P = layers.Lambda(lambda x: x[:, split_size*2:])(X_input)\n",
    "\n",
    "    head_num = 6\n",
    "    res = []\n",
    "    for head in range(head_num):\n",
    "        query_encoder = layers.Dense(dense_layer_sizes[0], activation='selu', kernel_regularizer = regularizers.l2(1e-6))\n",
    "        ans_encoder = layers.Dense(dense_layer_sizes[0], activation='selu', kernel_regularizer = regularizers.l2(1e-6))\n",
    "        \n",
    "        d_ratio = 0.6\n",
    "        \n",
    "        a, b = query_encoder(layers.Dropout(d_ratio)(A)), query_encoder(layers.Dropout(d_ratio)(B))\n",
    "        p = query_encoder(layers.Dropout(d_ratio)(P))\n",
    "        \n",
    "        amp = layers.Multiply()([a, p])\n",
    "        bmp = layers.Multiply()([b, p])\n",
    "        \n",
    "        asp = layers.Lambda(lambda v: v[0] - v[1])([p, a])\n",
    "        bsp = layers.Lambda(lambda v: v[0] - v[1])([p, b])        \n",
    "        \n",
    "        ia = layers.Concatenate()([a, p, amp, asp])\n",
    "        ib = layers.Concatenate()([b, p, bmp, bsp])\n",
    "        nli_encoder = layers.Dense(dense_layer_sizes[0], activation='selu')\n",
    "        ia, ib = nli_encoder(ia), nli_encoder(ib)\n",
    "        \n",
    "        out = layers.Concatenate()([ia, ib])\n",
    "        res.append(out)\n",
    "    \n",
    "    res = layers.Add()(res)\n",
    "    res = layers.Dropout(0.8)(res)\n",
    "    X = layers.Dense(dense_layer_sizes[0], name = 'dense0')(res)\n",
    "    X = layers.BatchNormalization(name = 'bn0')(X)\n",
    "    X = layers.Activation('relu')(X)\n",
    "    X = layers.Dropout(dropout_rate, seed = 7)(X)\n",
    "\n",
    "    # Output layer\n",
    "    X = layers.Dense(3, name = 'output', kernel_regularizer = regularizers.l2(lambd))(X)\n",
    "    X = layers.Activation('softmax')(X)\n",
    "\n",
    "    # Create model\n",
    "    model = models.Model(input = X_input, output = X, name = \"classif_model\")\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n------------------------------\\nFor the model bert-large-uncased-seq256-19\\nCV mean score: 0.3441, std: 0.0215.\\n[0.3307294825242308, 0.3187784074490988, 0.34923320252765083, 0.3821268940718645, 0.3397000497226646]\\nTest score: 0.32621788223567494\\n------------------------------\\n'"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def build_mlp_model_for_larget_bert(input_shape, split_size):\n",
    "    X_input = layers.Input(input_shape)\n",
    "\n",
    "    # First dense layer\n",
    "    A = layers.Lambda(lambda x: x[:, :split_size])(X_input)\n",
    "    B = layers.Lambda(lambda x: x[:, split_size:split_size*2])(X_input)\n",
    "    P = layers.Lambda(lambda x: x[:, split_size*2:])(X_input)\n",
    "\n",
    "    head_num = 6\n",
    "    res = []\n",
    "    for head in range(head_num):\n",
    "        query_encoder = layers.Dense(dense_layer_sizes[0], activation='selu', kernel_regularizer = regularizers.l2(1e-6))\n",
    "        ans_encoder = layers.Dense(dense_layer_sizes[0], activation='selu', kernel_regularizer = regularizers.l2(1e-6))\n",
    "        \n",
    "        d_ratio = 0.7\n",
    "        \n",
    "        a, b = query_encoder(layers.Dropout(d_ratio)(A)), query_encoder(layers.Dropout(d_ratio)(B))\n",
    "        p = query_encoder(layers.Dropout(d_ratio)(P))\n",
    "        \n",
    "        amp = layers.Multiply()([a, p])\n",
    "        bmp = layers.Multiply()([b, p])\n",
    "        \n",
    "        asp = layers.Lambda(lambda v: v[0] - v[1])([p, a])\n",
    "        bsp = layers.Lambda(lambda v: v[0] - v[1])([p, b])        \n",
    "        \n",
    "        ia = layers.Concatenate()([a, p, amp, asp])\n",
    "        ib = layers.Concatenate()([b, p, bmp, bsp])\n",
    "        nli_encoder = layers.Dense(dense_layer_sizes[0], activation='selu')\n",
    "        ia, ib = nli_encoder(ia), nli_encoder(ib)\n",
    "        \n",
    "        out = layers.Concatenate()([ia, ib])\n",
    "        res.append(out)\n",
    "    \n",
    "    res = layers.Concatenate()(res)\n",
    "    res = layers.Dropout(0.85)(res)\n",
    "    X = layers.Dense(dense_layer_sizes[0], name = 'dense0')(res)\n",
    "    X = layers.BatchNormalization(name = 'bn0')(X)\n",
    "    X = layers.Activation('relu')(X)\n",
    "    X = layers.Dropout(dropout_rate, seed = 7)(X)\n",
    "    \n",
    "    # Output layer\n",
    "    X = layers.Dense(3, name = 'output', kernel_regularizer = regularizers.l2(lambd))(X)\n",
    "    X = layers.Activation('softmax')(X)\n",
    "\n",
    "    # Create model\n",
    "    model = models.Model(input = X_input, output = X, name = \"classif_model\")\n",
    "    return model\n",
    "\n",
    "'''\n",
    "------------------------------\n",
    "For the model bert-large-uncased-seq256-19\n",
    "CV mean score: 0.3441, std: 0.0215.\n",
    "[0.3307294825242308, 0.3187784074490988, 0.34923320252765083, 0.3821268940718645, 0.3397000497226646]\n",
    "Test score: 0.32621788223567494\n",
    "------------------------------\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras.backend as K\n",
    "\n",
    "def base_nli_model(input_shape, split_size):\n",
    "    X_input = layers.Input(input_shape)\n",
    "\n",
    "    # First dense layer\n",
    "    A = layers.Lambda(lambda x: x[:, :split_size])(X_input)\n",
    "    B = layers.Lambda(lambda x: x[:, split_size:split_size*2])(X_input)\n",
    "    P = layers.Lambda(lambda x: x[:, split_size*2:])(X_input)\n",
    "\n",
    "    X1 = layers.Dropout(0.6, seed = 7)(A)\n",
    "    X2 = layers.Dropout(0.6, seed = 7)(B)\n",
    "    Y = layers.Dropout(0.6, seed = 7)(P)    \n",
    "    \n",
    "    def interaction(a, b):\n",
    "        sub = layers.Lambda(lambda a: K.abs(a[0] - a[1]))([a, b])\n",
    "        mult = layers.Lambda(lambda a: a[0] * a[1])([a, b])\n",
    "        return layers.Concatenate()([a, b, sub, mult,])    \n",
    "    \n",
    "    word_encoder = layers.Dense(512, activation='selu')\n",
    "    X1 = word_encoder(X1)\n",
    "    X2 = word_encoder(X2)\n",
    "    Y = word_encoder(Y)\n",
    "\n",
    "    I_X1_Y = interaction(X1, Y)\n",
    "    I_X2_Y = interaction(X2, Y)\n",
    "\n",
    "    dense_encoder = layers.Dense(128, activation='selu')\n",
    "\n",
    "    I_X1_Y = layers.Dropout(0.75)(dense_encoder(I_X1_Y))\n",
    "    I_X2_Y = layers.Dropout(0.75)(dense_encoder(I_X2_Y))\n",
    "    features = layers.Concatenate()([I_X1_Y, I_X2_Y])\n",
    "\n",
    "    X = layers.Dense(dense_layer_sizes[0], name = 'dense0')(features)\n",
    "    X = layers.BatchNormalization(name = 'bn0')(X)\n",
    "    X = layers.Activation('relu')(X)\n",
    "    X = layers.Dropout(dropout_rate, seed = 7)(X)\n",
    "\n",
    "    # Output layer\n",
    "    X = layers.Dense(3, name = 'output', kernel_regularizer = regularizers.l2(lambd))(X)\n",
    "    X = layers.Activation('softmax')(X)\n",
    "\n",
    "    # Create model\n",
    "    model = models.Model(input = X_input, output = X, name = \"classif_model\")\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_tag, model_func,\n",
    "                dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "                oof_folder='oof/', pred_folder='test_outputs/'):\n",
    "\n",
    "    tag = tag + str(layer) # follow the original naming style\n",
    "    dev_filename = tag + dev_filename\n",
    "    val_filename = tag + val_filename\n",
    "    test_filename = tag + test_filename\n",
    "    \n",
    "    development = pd.read_json(os.path.join(dev_folder_path, dev_filename))\n",
    "    X_development, Y_development = parse_json(development, embedding_size)\n",
    "\n",
    "    # We want predictions for all development rows. So instead of removing rows, make them 0\n",
    "    remove_development = [row for row in range(len(X_development)) if np.sum(np.isnan(X_development[row].reshape(-1)))]\n",
    "    X_development[remove_development] = np.zeros((3 * embedding_size))\n",
    "\n",
    "    prediction = np.zeros((len(X_development), 3)) # testing predictions\n",
    "\n",
    "    for fold_n in range(n_fold):\n",
    "        # split training and validation data\n",
    "        print('Make predictions for', fold_n, 'started at', time.ctime())\n",
    "        classif_model = model_func([X_development.shape[-1]], split_size=embedding_size)\n",
    "        classif_model.load_weights(os.path.join(checkpoint_path, model_tag + tag + str(fold_n) + '.pt'))\n",
    "\n",
    "        pred = classif_model.predict(x=X_development, verbose=0)\n",
    "        prediction += pred\n",
    "    \n",
    "    prediction /= n_fold\n",
    "    # prediction[remove_development] = [0.4, 0.4, 0.2] This is useless for prediction stage\n",
    "\n",
    "    submission = pd.read_csv(\"data/sample_submission_stage_2.csv\", index_col = \"ID\")\n",
    "    submission[\"A\"] = prediction[:,0]\n",
    "    submission[\"B\"] = prediction[:,1]\n",
    "    submission[\"NEITHER\"] = prediction[:,2]\n",
    "    submission.to_csv(pred_folder + model_tag + tag + \".csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 21:53:46 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:46: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 21:53:49 2019\n",
      "Make predictions for 2 started at Fri Apr 19 21:53:52 2019\n",
      "Make predictions for 3 started at Fri Apr 19 21:53:56 2019\n",
      "Make predictions for 4 started at Fri Apr 19 21:53:59 2019\n",
      "Make predictions for 5 started at Fri Apr 19 21:54:03 2019\n",
      "Make predictions for 6 started at Fri Apr 19 21:54:07 2019\n",
      "Make predictions for 0 started at Fri Apr 19 21:54:14 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:44: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 21:54:16 2019\n",
      "Make predictions for 2 started at Fri Apr 19 21:54:18 2019\n",
      "Make predictions for 3 started at Fri Apr 19 21:54:20 2019\n",
      "Make predictions for 4 started at Fri Apr 19 21:54:22 2019\n",
      "Make predictions for 5 started at Fri Apr 19 21:54:24 2019\n",
      "Make predictions for 6 started at Fri Apr 19 21:54:26 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_base\"\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-base-uncased-seq512-\"\n",
    "pred_tag = \"nli-mh-\"\n",
    "model_func = build_mlp_model_for_base_bert\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag, model_func=model_func)\n",
    "\n",
    "pred_tag = \"bnli-mh-\"\n",
    "model_func = base_nli_model\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag, model_func=model_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 21:54:32 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:46: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 21:54:36 2019\n",
      "Make predictions for 2 started at Fri Apr 19 21:54:40 2019\n",
      "Make predictions for 3 started at Fri Apr 19 21:54:43 2019\n",
      "Make predictions for 4 started at Fri Apr 19 21:54:48 2019\n",
      "Make predictions for 5 started at Fri Apr 19 21:54:52 2019\n",
      "Make predictions for 6 started at Fri Apr 19 21:54:56 2019\n",
      "Make predictions for 0 started at Fri Apr 19 21:55:03 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:44: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 21:55:06 2019\n",
      "Make predictions for 2 started at Fri Apr 19 21:55:09 2019\n",
      "Make predictions for 3 started at Fri Apr 19 21:55:12 2019\n",
      "Make predictions for 4 started at Fri Apr 19 21:55:14 2019\n",
      "Make predictions for 5 started at Fri Apr 19 21:55:18 2019\n",
      "Make predictions for 6 started at Fri Apr 19 21:55:20 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_base_cased\"\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-base-cased-seq512-\"\n",
    "pred_tag = \"nli-mh-\"\n",
    "model_func = build_mlp_model_for_base_bert\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag, model_func=model_func)\n",
    "\n",
    "pred_tag = \"bnli-mh-\"\n",
    "model_func = base_nli_model\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag, model_func=model_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 21:55:28 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:46: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 21:55:32 2019\n",
      "Make predictions for 2 started at Fri Apr 19 21:55:37 2019\n",
      "Make predictions for 3 started at Fri Apr 19 21:55:42 2019\n",
      "Make predictions for 4 started at Fri Apr 19 21:55:46 2019\n",
      "Make predictions for 5 started at Fri Apr 19 21:55:51 2019\n",
      "Make predictions for 6 started at Fri Apr 19 21:55:57 2019\n",
      "Make predictions for 0 started at Fri Apr 19 21:56:06 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:44: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 21:56:10 2019\n",
      "Make predictions for 2 started at Fri Apr 19 21:56:14 2019\n",
      "Make predictions for 3 started at Fri Apr 19 21:56:17 2019\n",
      "Make predictions for 4 started at Fri Apr 19 21:56:21 2019\n",
      "Make predictions for 5 started at Fri Apr 19 21:56:24 2019\n",
      "Make predictions for 6 started at Fri Apr 19 21:56:28 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_big\"\n",
    "embedding_size = 1024\n",
    "layer = 19\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-large-uncased-seq300-\"\n",
    "pred_tag = \"nli-mh-\"\n",
    "model_func = build_mlp_model_for_larget_bert\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag, model_func=model_func)\n",
    "\n",
    "pred_tag = \"bnli-mh-\"\n",
    "model_func = base_nli_model\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag, model_func=model_func)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 21:56:37 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:46: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 21:56:42 2019\n",
      "Make predictions for 2 started at Fri Apr 19 21:56:47 2019\n",
      "Make predictions for 3 started at Fri Apr 19 21:56:53 2019\n",
      "Make predictions for 4 started at Fri Apr 19 21:56:59 2019\n",
      "Make predictions for 5 started at Fri Apr 19 21:57:05 2019\n",
      "Make predictions for 6 started at Fri Apr 19 21:57:11 2019\n",
      "Make predictions for 0 started at Fri Apr 19 21:57:21 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:44: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=Tensor(\"in..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 21:57:26 2019\n",
      "Make predictions for 2 started at Fri Apr 19 21:57:30 2019\n",
      "Make predictions for 3 started at Fri Apr 19 21:57:35 2019\n",
      "Make predictions for 4 started at Fri Apr 19 21:57:40 2019\n",
      "Make predictions for 5 started at Fri Apr 19 21:57:44 2019\n",
      "Make predictions for 6 started at Fri Apr 19 21:57:49 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_big_cased\"\n",
    "embedding_size = 1024\n",
    "layer = 18\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-large-cased-seq300-\"\n",
    "pred_tag = \"nli-mh-\"\n",
    "model_func = build_mlp_model_for_larget_bert\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag, model_func=model_func)\n",
    "\n",
    "pred_tag = \"bnli-mh-\"\n",
    "model_func = base_nli_model\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag, model_func=model_func)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Predictons for stage 1.4\n",
    "\n",
    "### Prepare Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "nlp = spacy.load('en_core_web_lg')\n",
    "\n",
    "# Load the GAP data\n",
    "\n",
    "data = pd.concat([pd.read_csv('fixed_test_stage_2.tsv', sep='\\t'),\n",
    "                  pd.read_csv('gap-validation.tsv', sep='\\t'),\n",
    "                  pd.read_csv('gap-test.tsv', sep='\\t'), \n",
    "                 ], ignore_index = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2959724259fd4706a7523b685f3087fa"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\numpy\\core\\fromnumeric.py:2957: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\numpy\\core\\_methods.py:80: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As wrongly disqualified: 36 , Bs wrongly disqualified: 44 \n",
      "\n",
      "validation-12 (12370): A>\n",
      "\tNicole flirts with Charlie and gives him her phone number. She later\n",
      "\ttruants from school and visits Charlie. Nicole later arranges a date with\n",
      "\tCharlie, telling Michael she feels too ill to go to school. When\n",
      "\tNicole kisses Charlie, Annie Sobacz (Reanne Farley) stops\n",
      "\ther and tells Charlie that Nicole is a school girl.\n",
      "\n",
      "Nicole[PROPN] >nsubj> flirts | flirts[VERB] >ROOT> flirts | with[ADP] >prep> flirts | \n",
      "Charlie[PROPN] >pobj> with | and[CCONJ] >cc> flirts | gives[VERB] >conj> flirts | \n",
      "him[PRON] >dative> gives | her[ADJ] >poss> number | phone[NOUN] >compound> number | \n",
      "number[NOUN] >dobj> gives | .[PUNCT] >punct> flirts | She[PRON] >nsubj> truants | \n",
      "later[ADV] >advmod> truants | truants[NOUN] >ROOT> truants | from[ADP] >prep> truants | \n",
      "school[NOUN] >pobj> from | and[CCONJ] >cc> truants | visits[VERB] >conj> truants | \n",
      "Charlie[PROPN] >dobj> visits | .[PUNCT] >punct> truants | Nicole[PROPN] >nsubj> arranges | \n",
      "later[ADV] >advmod> arranges | arranges[VERB] >ROOT> arranges | a[DET] >det> date | \n",
      "date[NOUN] >dobj> arranges | with[ADP] >prep> date | Charlie[PROPN] >pobj> with | \n",
      ",[PUNCT] >punct> arranges | telling[VERB] >advcl> arranges | Michael[PROPN] >dobj> telling | \n",
      "she[PRON] >nsubj> feels | feels[VERB] >ccomp> telling | too[ADV] >advmod> ill | \n",
      "ill[ADJ] >acomp> feels | to[PART] >aux> go | go[VERB] >xcomp> ill | \n",
      "to[ADP] >prep> go | school[NOUN] >pobj> to | .[PUNCT] >punct> arranges | \n",
      "When[ADV] >advmod> kisses | Nicole[PROPN] >nsubj> kisses | kisses[VERB] >advcl> stops | \n",
      "Charlie[PROPN] >dobj> kisses | ,[PUNCT] >punct> stops | Annie[PROPN] >compound> Sobacz | \n",
      "Sobacz[PROPN] >nsubj> stops | ([PUNCT] >punct> Sobacz | Reanne[PROPN] >compound> Farley | \n",
      "Farley[PROPN] >appos> Sobacz | )[PUNCT] >punct> Sobacz | stops[VERB] >ROOT> stops | \n",
      "her[PRON] >dobj> stops | and[CCONJ] >cc> stops | tells[VERB] >conj> stops | \n",
      "Charlie[PROPN] >dobj> tells | that[ADP] >mark> is | Nicole[PROPN] >nsubj> is | \n",
      "is[VERB] >ccomp> tells | a[DET] >det> girl | school[NOUN] >compound> girl | \n",
      "girl[NOUN] >attr> is | .[PUNCT] >punct> stops | \n",
      "\n"
     ]
    }
   ],
   "source": [
    "def domain(t):\n",
    "    while not t._.subj and not t._.poss and\\\n",
    "            not (t.dep_ == 'xcomp' and t.head._.obj) and\\\n",
    "            t != t.head:\n",
    "        t = t.head\n",
    "    return t\n",
    "\n",
    "def ccom(t):\n",
    "    return [t2 for t2 in t.head._.d]\n",
    "\n",
    "# spacy extensions:\n",
    "#   doc._.to(offset) => t at text char offset\n",
    "#   t._.c => t's children (list)                     #   t._.d => t's descendents (list)\n",
    "#   t._.subj => t's subject else False               #   t._.obj => t's object else False\n",
    "#   t._.domain => t's syntactic domain               #   t._.ccom => t's c-command domain\n",
    "\n",
    "spacy.tokens.doc.Doc.set_extension(\n",
    "    'to', method=lambda doc, offset: [t for t in doc if t.idx == offset][0], force=True)\n",
    "spacy.tokens.token.Token.set_extension(\n",
    "    'c', getter=lambda t: [c for c in t.children], force=True)\n",
    "spacy.tokens.token.Token.set_extension(\n",
    "    'd', getter=lambda t: [c for c in t.sent if t in list(c.ancestors)], force=True)\n",
    "spacy.tokens.token.Token.set_extension(\n",
    "    'subj', getter=lambda t: ([c for c in t._.c if c.dep_.startswith('nsubj')] + [False])[0], force=True)\n",
    "spacy.tokens.token.Token.set_extension(\n",
    "    'obj', getter=lambda t: ([c for c in t._.c if c.dep_.startswith('dobj')] + [False])[0], force=True)\n",
    "spacy.tokens.token.Token.set_extension(\n",
    "    'poss', getter=lambda t: ([c for c in t._.c if c.dep_.startswith('poss')] + [False])[0], force=True)\n",
    "spacy.tokens.token.Token.set_extension(\n",
    "    'span', method=lambda t, t2: t.doc[t.i:t2.i] if t.i < t2.i else t.doc[t2.i:t.i], force=True)\n",
    "spacy.tokens.token.Token.set_extension('domain', getter=domain, force=True)\n",
    "spacy.tokens.token.Token.set_extension('ccom', getter=ccom, force=True)\n",
    "\n",
    "def applyDisq(condition, candidates, candidate_dict, debug = False):\n",
    "    badnames = sum([nameset(c, candidate_dict) for c in candidates if c in condition[0]], [])\n",
    "    badcands = [c for c in candidates if c.text in badnames]\n",
    "    if debug and len(badcands) > 0: print('Disqualified:', badcands, '<', condition[1])\n",
    "    return [c for c in candidates if c not in badcands]\n",
    "\n",
    "# Apply a list of disqualifying conditions\n",
    "def applyDisqs(conditions, candidates, candidate_dict, debug = False):\n",
    "    for condition in conditions:\n",
    "        if len(candidates) < 1: return candidates\n",
    "        candidates = applyDisq(condition, candidates, candidate_dict, debug)\n",
    "    return candidates\n",
    "\n",
    "# Pass the list of disqualifying conditions for possessive pronouns (his, her)\n",
    "def disqGen(t, candidates, candidate_dict, debug = False):\n",
    "    conds = [(t._.ccom,\n",
    "             \"disqualify candidates c-commanded by genpn; e.g. e.g. *Julia read his_i book about John_i's life.\"),\n",
    "             ([t2 for t2 in candidates if t in t2._.ccom and t2.head.dep_ == 'appos'],\n",
    "             \"disqualify candidates modified by an appositive with genpn; e.g. *I wanted to see John_i, his_i father.\")\n",
    "            ]\n",
    "    return applyDisqs(conds, candidates, candidate_dict, debug)\n",
    "\n",
    "# Pass the list of list of disqualifying conditions for other pronouns\n",
    "def disqOthers(t, candidates, candidate_dict, debug = False):\n",
    "    conds = [([t2 for t2 in t._.ccom if t2.i > t.i],\n",
    "             \"disqualify candidates c-commanded by pn, unless they were preposed;\\\n",
    "             e.g. *He_i cried before John_i laughed. vs. Before John_i laughed, he_i cried.\"),\n",
    "             ([t2 for t2 in candidates if t in t2._.ccom and t2._.domain == t._.domain\n",
    "              and not (t.head.text == 'with' and t.head.head.lemma_ == 'take')],\n",
    "             \"disqualify candidates that c-command pn, unless in different domain;\\\n",
    "             e.g. Mary said that *John_i hit him_i. vs. John_i said that Mary hit him_i;\\\n",
    "             random hard-coded exception: `take with'\"),\n",
    "             ([t2 for t2 in candidates if t2._.domain.dep_ == 'xcomp' and t2._.domain.head._.obj and t2 == t2._.domain.head._.obj],\n",
    "             \"for xcomps with subjects parsed as upstairs dobj, disallow coref with that dobj;\\\n",
    "             e.g. *Mary wanted John_i to forgive him_i.\")\n",
    "            ]\n",
    "    return applyDisqs(conds, candidates, candidate_dict, debug)\n",
    "\n",
    "# Decide whether possessive or not and call appropriate function\n",
    "def disq(t, candidates, candidate_dict, debug = False):\n",
    "    func = disqGen if t.dep_ == 'poss' else disqOthers\n",
    "    candidates = func(t, candidates, candidate_dict, debug)\n",
    "    return candidates\n",
    "\n",
    "def find_head(w, wo, doc):\n",
    "    t = False; backtrack = 0\n",
    "    while not t:\n",
    "        try:\n",
    "            t = doc._.to(wo)\n",
    "        except IndexError:\n",
    "            wo -= 1; backtrack += 1\n",
    "    while t.dep_ == 'compound' and t.head.idx >= wo and t.head.idx < len(w) + wo + backtrack: t = t.head\n",
    "    return t\n",
    "\n",
    "# Returns subsequences of a name\n",
    "def subnames(name):\n",
    "    if type(name) != str: name = candidate_dict[name]\n",
    "    parts = name.split(' ')\n",
    "    subnames_ = []\n",
    "    for i in range(len(parts)): \n",
    "        for j in range(i + 1, len(parts) + 1): \n",
    "            sub = ' '.join(parts[i:j])\n",
    "            if len(sub) > 2: subnames_.append(sub)\n",
    "    return subnames_\n",
    "\n",
    "# Returns subsequences of a name unless potentially ambiguous (if another candidate picks out same subsequence)\n",
    "def nameset(name, candidate_dict):\n",
    "    if type(name) != str: name = candidate_dict[name]\n",
    "    subnames_ = [sn for sn in subnames(name)]\n",
    "    return [c for c in subnames_ if c not in sum([subnames(c) for c in candidate_dict.values() \n",
    "                                                  if c not in subnames_ and name not in subnames(c)], [])]\n",
    "\n",
    "# Given the original candidate dict and the final candidate list, returns new dict grouping putative candidate instances under a single key\n",
    "def candInstances(candidates, candidate_dict):\n",
    "    candidates_by_name = {}\n",
    "    for c in sorted(candidates, key = lambda c: len(candidate_dict[c]), reverse = True):\n",
    "        name = candidate_dict[c]\n",
    "        for name2 in candidates_by_name.keys():\n",
    "            if name in nameset(name2, candidate_dict): name = name2; break\n",
    "        candidates_by_name[name] = candidates_by_name.get(name, []) + [c]\n",
    "    return candidates_by_name\n",
    "\n",
    "import gender_guesser.detector as gender # oops\n",
    "gd = gender.Detector()\n",
    "\n",
    "# Needed to prune candidate dict-- removes non-provided candidates that don't match in most common gender with pn\n",
    "def filterGender(candidates_by_name, a, b, pn):\n",
    "    badnames = []\n",
    "    gender = 'female' if pn in ['She', 'she', 'her', 'Her'] else 'male'\n",
    "    for name in candidates_by_name.keys():\n",
    "        if a in subnames(name) or b in subnames(name): continue\n",
    "        genderii = gd.get_gender(name.split(' ')[0])\n",
    "        if gender == 'male' and genderii == 'female': badnames += [name]; continue\n",
    "        if gender == 'female' and genderii == 'male': badnames += [name]; continue\n",
    "    for name in badnames: candidates_by_name.pop(name)\n",
    "    return candidates_by_name\n",
    "\n",
    "from urllib.parse import unquote\n",
    "import re\n",
    "\n",
    "# Authors' metric 1: Does the Wikipedia url contain the candidate's name?\n",
    "def urlMatch(a, b, url, candidate_dict):\n",
    "    url = re.sub('[^\\x00-\\x7F]', '*', unquote(url.split('/')[-1])).replace('_', ' ').lower()\n",
    "    return {'a_url': (sorted([len(n.split(' ')) for n in nameset(a.lower(), candidate_dict) if n in nameset(url, candidate_dict)], reverse = True) + [0])[0],\n",
    "            'b_url': (sorted([len(n.split(' ')) for n in nameset(b.lower(), candidate_dict) if n in nameset(url, candidate_dict)], reverse = True) + [0])[0]}\n",
    "\n",
    "# Authors' metric 2: When pn is subject or object, does the candidate match?\n",
    "def parallel(t1, t2):\n",
    "    if t1.dep_.startswith('nsubj'): return t2.dep_.startswith('nsubj')\n",
    "    if t1.dep_.startswith('dobj'): return t2.dep_.startswith('dobj')\n",
    "    if t1.dep_.startswith('dative'): return t2.dep_.startswith('dative')\n",
    "    return False\n",
    "\n",
    "# Depth from a node to a parent node\n",
    "def depthTo(t1, t2):\n",
    "    depth = 0\n",
    "    while t1 != t2 and t1 != t1.head:\n",
    "        t1 = t1.head\n",
    "        depth += 1\n",
    "    return depth\n",
    "\n",
    "# Syntactic distance within a single tree\n",
    "def nodeDist(t1, t2):\n",
    "    if t1 == t2: return 0\n",
    "    if t2 in t1._.d: return depthTo(t2, t1)\n",
    "    if t1 in t2._.d: return depthTo(t1, t2)\n",
    "    t = t1\n",
    "    while t1 not in t._.d or t2 not in t._.d and t != t.head: t = t.head\n",
    "    return depthTo(t1, t) + depthTo(t2, t)\n",
    "\n",
    "# Authors' metric 3: Syntactic distance (within or across trees)\n",
    "def synDist(t, pn, doc, debug = False):\n",
    "    doc_sents = list(doc.sents)\n",
    "    sspan = doc_sents.index(pn.sent) - doc_sents.index(t.sent)\n",
    "    if sspan == 0: # same sentence\n",
    "        dist = nodeDist(t, pn)\n",
    "    else: # different sentence\n",
    "        dist = nodeDist(pn, doc_sents[doc_sents.index(pn.sent)].root) + nodeDist(t, doc_sents[doc_sents.index(t.sent)].root) # dist from two roots\n",
    "    if debug: \n",
    "        print('pn dist:', nodeDist(pn, doc_sents[doc_sents.index(pn.sent)].root), '; t dist:',\n",
    "              nodeDist(t, doc_sents[doc_sents.index(t.sent)].root), '; span:', sspan)\n",
    "    sspan = abs(sspan) * 1 if sspan >= 0 else abs(sspan) * 1.3 # less local if not preceding\n",
    "    return dist + sspan# * 0.7\n",
    "\n",
    "# Character distance\n",
    "def charDist(t1, t2):\n",
    "    if t2.idx > t1.idx:\n",
    "        return t2.idx - t1.idx + len(t1.text)\n",
    "    else:\n",
    "        return (t1.idx - t2.idx + len(t2.text)) * 1.3\n",
    "\n",
    "# Theta prominence: assign a 0.1 to 1 score based on dep role of candidate -- strong feature\n",
    "def thetaProminence(t, mult = 1, debug = False):\n",
    "    while t.dep_ == 'compound': t = t.head\n",
    "    if debug: print('t dep_:', t.dep_)\n",
    "    if t.dep_ == 'pobj': mult = 1.3 if t.head.i < t.head.head.i else 1\n",
    "    if t._.domain.dep_ == 'advcl': mult = 1.3 if t.head.i < t._.domain.head.i else 1\n",
    "    if t.dep_.startswith('nsubj'): score = 1\n",
    "    elif t.dep_.startswith('dobj'): score = 0.8\n",
    "    elif t.dep_.startswith('dative'): score = 0.6\n",
    "    elif t.dep_.startswith('pobj'): score = 0.4\n",
    "    elif t.dep_.startswith('poss'): score = 0.3\n",
    "    else: score = 0.1\n",
    "    if debug: print('mult:', mult, '; score:', score)\n",
    "    return min(1, score * mult)\n",
    "\n",
    "# Computes these metrics for each candidate, and returns, for each group of instances (A instances, B instances,\\\n",
    "# other instances), either the sum, or the highest difference from the mean\n",
    "def score(label, candidates_by_name, a_cand, b_cand, func, minsc = None, method = 'sum'):\n",
    "    if method == 'sum':\n",
    "        scores = {name: sum([func(t) for t in tokens]) for name, tokens in candidates_by_name.items()}\n",
    "    elif method == 'meandiff':\n",
    "        mean = np.mean(sum([[func(t) for t in tokens] for tokens in candidates_by_name.values()], []))\n",
    "        scores = {name: mean - min([func(t) for t in tokens]) for name, tokens in candidates_by_name.items()}\n",
    "    sca = scores[a_cand] if a_cand else minsc\n",
    "    scb = scores[b_cand] if b_cand else minsc\n",
    "    screst = [v for n, v in scores.items() if n != a_cand and n != b_cand]\n",
    "    if method == 'sum':\n",
    "        screst = sum(screst) if len(screst) > 0 else minsc\n",
    "    elif method == 'meandiff':\n",
    "        screst = max(screst) if len(screst) > 0 else minsc\n",
    "    return {'a_' + label: sca, 'b_' + label: scb, 'n_' + label: screst}\n",
    "\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "\n",
    "# Load a rowfull of data\n",
    "def load_row(data, i):\n",
    "    return tuple(data.iloc[i])\n",
    "\n",
    "# Row by row, populate features\n",
    "def annotateSet(data, minsc = None, debug = False):\n",
    "    \n",
    "    annotated_data = pd.DataFrame() # init placeholder df\n",
    "    row_batch = []\n",
    "\n",
    "    for i in tqdm(range(annotated_data.shape[0], data.shape[0])):\n",
    "        id, text, pn, pno, a, ao, ag, b, bo, bg, url = load_row(data, i)        \n",
    "        doc = nlp(text) # parse text into doc\n",
    "        try:\n",
    "            pnt, at, bt = (doc._.to(pno), find_head(a, ao, doc), find_head(b, bo, doc)) # get the tokens that correspond to offsets\n",
    "        except:\n",
    "            # There is an error which has already been reported by \n",
    "            # https://www.kaggle.com/c/gendered-pronoun-resolution/discussion/89830#latest-518973\n",
    "            # my fixing is simply replacing the `)` with a space ' ', so 'he' would match the offset and spacy also can work.\n",
    "            doc = nlp(text.replace(')', ' '))\n",
    "            pnt, at, bt = (doc._.to(pno), find_head(a, ao, doc), find_head(b, bo, doc))\n",
    "            \n",
    "        pnt, at, bt = (doc._.to(pno), find_head(a, ao, doc), find_head(b, bo, doc)) # get the tokens that correspond to offsets\n",
    "        candidate_dict = {e.root: re.sub('\\'s$', '', e.text) for e in [e for e in doc.ents if e.root.ent_type_ == 'PERSON']} # first get every PERSON ent as candidate\n",
    "        candidate_dict.update({c.root: re.sub('\\'s$', '', c.text) for c in doc.noun_chunks if c.root.pos_ == 'PROPN' and c.text in sum([subnames(n) for n in candidate_dict.values()], []) and\n",
    "                               c.root not in candidate_dict.keys()}) # get some missed ones by looking at noun chunks with PROPN roots whose text match part of a candidate but are not already in list\n",
    "        candidate_dict.update({t: w for t, w in [(at, a), (bt, b)]}) # add provided cands, overwriting in the process\n",
    "\n",
    "        candidates = disq(pnt, list(candidate_dict.keys()), candidate_dict, debug = False)\n",
    "        candidates_by_name = candInstances(candidates, candidate_dict)\n",
    "        candidates_by_name = filterGender(candidates_by_name, a, b, pn)\n",
    "        a_cand = ([name for name, tokens in candidates_by_name.items() if at in tokens] + [False])[0]\n",
    "        b_cand = ([name for name, tokens in candidates_by_name.items() if bt in tokens] + [False])[0]\n",
    "    \n",
    "        # init row dict\n",
    "        features = {'id': id, 'label': 0 if ag else 1 if bg else 2}\n",
    "        # eliminated or not\n",
    "        features.update({'a_out': 0 if a_cand else 1, 'b_out': 0 if b_cand else 1})\n",
    "        # url match or not\n",
    "        features.update(urlMatch(a, b, url, candidate_dict))\n",
    "        # c-command or not\n",
    "        features.update({'a_cc': 1 if a_cand and pnt in at._.ccom else 0, 'b_cc': 1 if b_cand and pnt in bt._.ccom else 0})\n",
    "        # parallelism score\n",
    "        features.update(score('par', candidates_by_name, a_cand, b_cand, lambda t: parallel(t, pnt), minsc = minsc))\n",
    "        # theta prominence score\n",
    "        features.update(score('th', candidates_by_name, a_cand, b_cand, thetaProminence, minsc = minsc))\n",
    "        # syntactic distance score\n",
    "        features.update(score('loc', candidates_by_name, a_cand, b_cand, lambda t: synDist(t, pnt, doc), method='meandiff', minsc = minsc))\n",
    "        # number of candidates left\n",
    "        features.update({'n_cands': len(candidates_by_name)})\n",
    "        # char dist\n",
    "        features.update(score('cloc', candidates_by_name, a_cand, b_cand, lambda t: charDist(t, pnt), method='meandiff', minsc = minsc))\n",
    "\n",
    "        if debug: print(id, '>', 'a:', 1 if ag else 0, 'b:', 1 if bg else 0, features)\n",
    "        row_batch += [features]\n",
    "\n",
    "    # add rows to placeholder df\n",
    "    if annotated_data.shape[0] != data.shape[0]: annotated_data = annotated_data.append(row_batch, ignore_index = True)\n",
    "    \n",
    "    return annotated_data\n",
    "\n",
    "# Readable rows\n",
    "\n",
    "from textwrap import TextWrapper\n",
    "wrapper = TextWrapper(width=75)\n",
    "\n",
    "def style(w, wstyle):\n",
    "    if wstyle == None: return '{}{}{}'.format('\\033[1m', w, '\\033[0m') # bold\n",
    "    elif wstyle == True: return '{}{}{}'.format('\\033[92m', w, '\\033[0m') # green\n",
    "    elif wstyle == False: return '{}{}{}'.format('\\033[91m', w, '\\033[0m') # red\n",
    "\n",
    "def readable_rows(data, range):\n",
    "    rows = []\n",
    "    for i in range:\n",
    "        wois = sorted([(data.iloc[i]['Pronoun'], data.iloc[i]['Pronoun-offset'], None),\\\n",
    "                   (data.iloc[i]['A'], data.iloc[i]['A-offset'], data.iloc[i]['A-coref']),\\\n",
    "                   (data.iloc[i]['B'], data.iloc[i]['B-offset'], data.iloc[i]['B-coref'])],\\\n",
    "                   key = lambda x: x[1]) # sort by offset\n",
    "        text = ''; ftext = data.iloc[i]['Text']; coffset = 0\n",
    "        for w, woffset, wstyle in wois:\n",
    "            text += ftext[coffset:woffset] + style(w, wstyle)\n",
    "            coffset += len(ftext[coffset:woffset]) + len(w)\n",
    "        text += ftext[coffset:]\n",
    "        rows += [(str(i), data.iloc[i]['ID'], 'A' if data.iloc[i]['A-coref'] else 'B' if data.iloc[i]['B-coref'] else 'N', text)]\n",
    "    return rows\n",
    "\n",
    "def print_readable_rows(data, range):\n",
    "    for index, id, target, text in readable_rows(data, range):\n",
    "        text = '\\n\\t'.join(wrapper.wrap(text = text))\n",
    "        print('{} ({}): {}>\\n\\t{}\\n'.format(id, index, target, text))\n",
    "        \n",
    "def print_tokens(doc):\n",
    "    for i in range(0, len(doc), 3):\n",
    "        for t in doc[i:i+3]:\n",
    "            print(\"{}[{}] >{}> {}\".format(t.text, t.pos_, t.dep_, t.head.text), end = ' | ')\n",
    "        print('')\n",
    "        \n",
    "annotated_data = annotateSet(data)\n",
    "\n",
    "misdisqs_a = annotated_data.loc[(annotated_data['a_out'] == 1) & (annotated_data['label'] == 0)]\n",
    "misdisqs_b = annotated_data.loc[(annotated_data['b_out'] == 1) & (annotated_data['label'] == 1)]\n",
    "\n",
    "print('As wrongly disqualified:', misdisqs_a.shape[0], ', Bs wrongly disqualified:', misdisqs_b.shape[0], '\\n')\n",
    "\n",
    "for i in misdisqs_a.index[:1]: # the first\n",
    "    print_readable_rows(data, [i])\n",
    "    id, text, pn, pno, a, ao, ag, b, bo, bg, url = load_row(data,i)\n",
    "    doc = nlp(text); print_tokens(doc)\n",
    "    print('')\n",
    "    \n",
    "train = annotated_data.loc[(annotated_data['id'].str.contains('test'))] # swap later\n",
    "valid = annotated_data.loc[(annotated_data['id'].str.contains('validation'))]\n",
    "test = annotated_data.loc[(annotated_data['id'].str.contains('development'))] # swap later\n",
    "\n",
    "# Note, the stage 2 test set should not be added into train_valid, or it might break the MinMaxScalar!!!\n",
    "train_valid = pd.concat([train, valid, test])\n",
    "\n",
    "answer_columns = ['label']\n",
    "excl = ['id']\n",
    "excl += ['a_out', 'b_out']\n",
    "feature_columns = [col for col in annotated_data.columns if col not in answer_columns and col not in excl]\n",
    "\n",
    "meta_features_train_valid = train_valid[feature_columns].fillna(0)\n",
    "meta_features_test = test[feature_columns].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:475: DataConversionWarning: Data with input dtype int64 was converted to float64 by MinMaxScaler.\n",
      "  warnings.warn(msg, DataConversionWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "meta_features_test shpae: (12359, 17)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "for col in meta_features_train_valid.columns:\n",
    "    std = MinMaxScaler()\n",
    "    meta_features_train_valid[col] = std.fit_transform(meta_features_train_valid[col].values.reshape(-1, 1))\n",
    "    meta_features_test[col] = std.transform(meta_features_test[col].values.reshape(-1, 1))\n",
    "    \n",
    "meta_features_test = meta_features_test.values\n",
    "print(\"meta_features_test shpae:\", meta_features_test.shape)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_mlp_model(input_shape, split_size, meta_features_shape):\n",
    "    X_input = layers.Input(input_shape)\n",
    "    meta_input = layers.Input(meta_features_shape)\n",
    "\n",
    "    # First dense layer\n",
    "    A = layers.Lambda(lambda x: x[:, :split_size])(X_input)\n",
    "    B = layers.Lambda(lambda x: x[:, split_size:split_size*2])(X_input)\n",
    "    P = layers.Lambda(lambda x: x[:, split_size*2:])(X_input)\n",
    "\n",
    "    head_num = 6\n",
    "    res = []\n",
    "    for head in range(head_num):\n",
    "        query_encoder = layers.Dense(dense_layer_sizes[0], activation='selu', kernel_regularizer = regularizers.l2(1e-6))\n",
    "        ans_encoder = layers.Dense(dense_layer_sizes[0], activation='selu', kernel_regularizer = regularizers.l2(1e-6))\n",
    "        \n",
    "        d_ratio = 0.6\n",
    "        \n",
    "        a, b = query_encoder(layers.Dropout(d_ratio)(A)), query_encoder(layers.Dropout(d_ratio)(B))\n",
    "        p = query_encoder(layers.Dropout(d_ratio)(P))\n",
    "        \n",
    "        amp = layers.Multiply()([a, p])\n",
    "        bmp = layers.Multiply()([b, p])\n",
    "        \n",
    "        asp = layers.Lambda(lambda v: v[0] - v[1])([p, a])\n",
    "        bsp = layers.Lambda(lambda v: v[0] - v[1])([p, b])        \n",
    "        \n",
    "        ia = layers.Concatenate()([a, p, amp, asp])\n",
    "        ib = layers.Concatenate()([b, p, bmp, bsp])\n",
    "        nli_encoder = layers.Dense(dense_layer_sizes[0], activation='selu')\n",
    "        ia, ib = nli_encoder(ia), nli_encoder(ib)\n",
    "        \n",
    "        out = layers.Concatenate()([ia, ib])\n",
    "        res.append(out)\n",
    "    \n",
    "    res = layers.Add()(res)\n",
    "    res = layers.Dropout(0.8)(res)\n",
    "    \n",
    "    meta_out = layers.Dense(20, activation='selu')(meta_input)\n",
    "    meta_out = layers.Dropout(0.25)(meta_out)    \n",
    "    \n",
    "    res = layers.Concatenate()([res, meta_out])\n",
    "    \n",
    "    X = layers.Dense(dense_layer_sizes[0], name = 'dense0')(res)\n",
    "    X = layers.BatchNormalization(name = 'bn0')(X)\n",
    "    X = layers.Activation('relu')(X)\n",
    "    X = layers.Dropout(dropout_rate, seed = 7)(X)\n",
    "\n",
    "    # Output layer\n",
    "    X = layers.Dense(3, name = 'output', kernel_regularizer = regularizers.l2(lambd))(X)\n",
    "    X = layers.Activation('softmax')(X)\n",
    "\n",
    "    # Create model\n",
    "    model = models.Model(input = [X_input, meta_input], output = X, name = \"classif_model\")\n",
    "    return model\n",
    "\n",
    "\n",
    "def build_base_nli_model(input_shape, split_size, meta_features_shape):\n",
    "    X_input = layers.Input(input_shape)\n",
    "    meta_input = layers.Input(meta_features_shape)\n",
    "\n",
    "    # First dense layer\n",
    "    A = layers.Lambda(lambda x: x[:, :split_size])(X_input)\n",
    "    B = layers.Lambda(lambda x: x[:, split_size:split_size*2])(X_input)\n",
    "    P = layers.Lambda(lambda x: x[:, split_size*2:])(X_input)\n",
    "\n",
    "    query_encoder = layers.Dense(512, activation='selu', kernel_regularizer = regularizers.l2(1e-6))\n",
    "    ans_encoder = layers.Dense(512, activation='selu', kernel_regularizer = regularizers.l2(1e-6))\n",
    "\n",
    "    d_ratio = 0.6\n",
    "\n",
    "    a, b = query_encoder(layers.Dropout(d_ratio)(A)), query_encoder(layers.Dropout(d_ratio)(B))\n",
    "    p = query_encoder(layers.Dropout(d_ratio)(P))\n",
    "\n",
    "    amp = layers.Multiply()([a, p])\n",
    "    bmp = layers.Multiply()([b, p])\n",
    "\n",
    "    asp = layers.Lambda(lambda v: v[0] - v[1])([p, a])\n",
    "    bsp = layers.Lambda(lambda v: v[0] - v[1])([p, b])        \n",
    "\n",
    "    ia = layers.Concatenate()([a, p, amp, asp])\n",
    "    ib = layers.Concatenate()([b, p, bmp, bsp])\n",
    "    \n",
    "    nli_encoder = layers.Dense(128, activation='selu')\n",
    "    ia, ib = nli_encoder(ia), nli_encoder(ib)\n",
    "\n",
    "    features = layers.Concatenate()([ia, ib])    \n",
    "    features = layers.Dropout(0.75)(features)\n",
    "    \n",
    "    meta_out = layers.Dense(20, activation='selu')(meta_input)\n",
    "    meta_out = layers.Dropout(0.25)(meta_out)    \n",
    "    \n",
    "    res = layers.Concatenate()([features, meta_out])\n",
    "    \n",
    "    X = layers.Dense(dense_layer_sizes[0], name = 'dense0')(res)\n",
    "    X = layers.BatchNormalization(name = 'bn0')(X)\n",
    "    X = layers.Activation('relu')(X)\n",
    "    X = layers.Dropout(dropout_rate, seed = 7)(X)\n",
    "\n",
    "    # Output layer\n",
    "    X = layers.Dense(3, name = 'output', kernel_regularizer = regularizers.l2(lambd))(X)\n",
    "    X = layers.Activation('softmax')(X)\n",
    "\n",
    "    # Create model\n",
    "    model = models.Model(input = [X_input, meta_input], output = X, name = \"classif_model\")\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_tag, model_func, \n",
    "                dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "                oof_folder='oof/', pred_folder='outputs/'):\n",
    "\n",
    "    tag = tag + str(layer) # follow the original naming style\n",
    "    dev_filename = tag + dev_filename\n",
    "    val_filename = tag + val_filename\n",
    "    test_filename = tag + test_filename\n",
    "    \n",
    "    development = pd.read_json(os.path.join(dev_folder_path, dev_filename))\n",
    "    X_development, Y_development = parse_json(development, embedding_size)\n",
    "\n",
    "    remove_development = [row for row in range(len(X_development)) if np.sum(np.isnan(X_development[row].reshape(-1)))]\n",
    "    X_development[remove_development] = np.zeros((3 * embedding_size))\n",
    "\n",
    "    # Will predict probabilities for data from the gap-development file; initializing the predictions\n",
    "    prediction = np.zeros((len(X_development), 3)) # testing predictions\n",
    "    \n",
    "    for fold_n in range(n_fold):\n",
    "        # split training and validation data\n",
    "        print('Make predictions for', fold_n, 'started at', time.ctime())\n",
    "        # Define the model, re-initializing for each fold\n",
    "        classif_model = model_func([X_development.shape[-1]], split_size=embedding_size, meta_features_shape=[meta_features_test.shape[-1]])\n",
    "        classif_model.load_weights(os.path.join(checkpoint_path, model_tag + tag + str(fold_n) + '.pt'))\n",
    "\n",
    "        pred = classif_model.predict(x=[X_development, meta_features_test], verbose=0)\n",
    "        prediction += pred\n",
    "    \n",
    "    prediction /= n_fold\n",
    "    \n",
    "    submission = pd.read_csv(\"data/sample_submission_stage_2.csv\", index_col = \"ID\")\n",
    "    submission[\"A\"] = prediction[:,0]\n",
    "    submission[\"B\"] = prediction[:,1]\n",
    "    submission[\"NEITHER\"] = prediction[:,2]\n",
    "    submission.to_csv(pred_folder + model_tag + tag + \".csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 22:09:59 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:53: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 22:10:06 2019\n",
      "Make predictions for 2 started at Fri Apr 19 22:10:12 2019\n",
      "Make predictions for 3 started at Fri Apr 19 22:10:20 2019\n",
      "Make predictions for 4 started at Fri Apr 19 22:10:27 2019\n",
      "Make predictions for 5 started at Fri Apr 19 22:10:33 2019\n",
      "Make predictions for 6 started at Fri Apr 19 22:10:40 2019\n",
      "Make predictions for 0 started at Fri Apr 19 22:10:51 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:104: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 22:10:57 2019\n",
      "Make predictions for 2 started at Fri Apr 19 22:11:02 2019\n",
      "Make predictions for 3 started at Fri Apr 19 22:11:07 2019\n",
      "Make predictions for 4 started at Fri Apr 19 22:11:13 2019\n",
      "Make predictions for 5 started at Fri Apr 19 22:11:19 2019\n",
      "Make predictions for 6 started at Fri Apr 19 22:11:24 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_base\"\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-base-uncased-seq512-\"\n",
    "pred_tag = \"Hnli-mh-\"\n",
    "model_func = build_mlp_model\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_func=model_func,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)\n",
    "\n",
    "pred_tag = \"HBnli-mh-\"\n",
    "model_func = build_base_nli_model\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_func=model_func,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 22:11:34 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:53: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 22:11:41 2019\n",
      "Make predictions for 2 started at Fri Apr 19 22:11:49 2019\n",
      "Make predictions for 3 started at Fri Apr 19 22:11:56 2019\n",
      "Make predictions for 4 started at Fri Apr 19 22:12:05 2019\n",
      "Make predictions for 5 started at Fri Apr 19 22:12:12 2019\n",
      "Make predictions for 6 started at Fri Apr 19 22:12:20 2019\n",
      "Make predictions for 0 started at Fri Apr 19 22:12:32 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:104: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 22:12:39 2019\n",
      "Make predictions for 2 started at Fri Apr 19 22:12:45 2019\n",
      "Make predictions for 3 started at Fri Apr 19 22:12:52 2019\n",
      "Make predictions for 4 started at Fri Apr 19 22:12:58 2019\n",
      "Make predictions for 5 started at Fri Apr 19 22:13:05 2019\n",
      "Make predictions for 6 started at Fri Apr 19 22:13:12 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_base_cased\"\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-base-cased-seq512-\"\n",
    "pred_tag = \"Hnli-mh-\"\n",
    "model_func = build_mlp_model\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_func=model_func,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)\n",
    "\n",
    "pred_tag = \"HBnli-mh-\"\n",
    "model_func = build_base_nli_model\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_func=model_func,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 22:13:23 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:53: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 22:13:32 2019\n",
      "Make predictions for 2 started at Fri Apr 19 22:13:40 2019\n",
      "Make predictions for 3 started at Fri Apr 19 22:13:49 2019\n",
      "Make predictions for 4 started at Fri Apr 19 22:13:58 2019\n",
      "Make predictions for 5 started at Fri Apr 19 22:14:07 2019\n",
      "Make predictions for 6 started at Fri Apr 19 22:14:16 2019\n",
      "Make predictions for 0 started at Fri Apr 19 22:14:30 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:104: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 22:14:38 2019\n",
      "Make predictions for 2 started at Fri Apr 19 22:14:46 2019\n",
      "Make predictions for 3 started at Fri Apr 19 22:14:54 2019\n",
      "Make predictions for 4 started at Fri Apr 19 22:15:01 2019\n",
      "Make predictions for 5 started at Fri Apr 19 22:15:09 2019\n",
      "Make predictions for 6 started at Fri Apr 19 22:15:17 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_big\"\n",
    "embedding_size = 1024\n",
    "layer = 19\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-large-uncased-seq300-\"\n",
    "pred_tag = \"Hnli-mh-\"\n",
    "model_func = build_mlp_model\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_func=model_func,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)\n",
    "\n",
    "pred_tag = \"HBnli-mh-\"\n",
    "model_func = build_base_nli_model\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_func=model_func,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 22:15:30 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:53: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 22:15:40 2019\n",
      "Make predictions for 2 started at Fri Apr 19 22:15:50 2019\n",
      "Make predictions for 3 started at Fri Apr 19 22:16:00 2019\n",
      "Make predictions for 4 started at Fri Apr 19 22:16:10 2019\n",
      "Make predictions for 5 started at Fri Apr 19 22:16:20 2019\n",
      "Make predictions for 6 started at Fri Apr 19 22:16:30 2019\n",
      "Make predictions for 0 started at Fri Apr 19 22:16:46 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:104: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 22:16:55 2019\n",
      "Make predictions for 2 started at Fri Apr 19 22:17:03 2019\n",
      "Make predictions for 3 started at Fri Apr 19 22:17:12 2019\n",
      "Make predictions for 4 started at Fri Apr 19 22:17:21 2019\n",
      "Make predictions for 5 started at Fri Apr 19 22:17:30 2019\n",
      "Make predictions for 6 started at Fri Apr 19 22:17:39 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_big_cased\"\n",
    "embedding_size = 1024\n",
    "layer = 18\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-large-cased-seq300-\"\n",
    "pred_tag = \"Hnli-mh-\"\n",
    "model_func = build_mlp_model\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_func=model_func,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)\n",
    "\n",
    "pred_tag = \"HBnli-mh-\"\n",
    "model_func = build_base_nli_model\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_func=model_func,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predictions for stage 1.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_mlp_model(input_shape, split_size, meta_features_shape,):\n",
    "    X_input = layers.Input(input_shape)\n",
    "    \n",
    "    meta_input = layers.Input(meta_features_shape)\n",
    "    meta_out = layers.Dropout(0.1, seed = 7)(meta_input)\n",
    "    meta_out = layers.Dense(20, activation='selu')(meta_out)\n",
    "    \n",
    "    X = layers.Dropout(dropout_rate, seed = 7)(X_input)\n",
    "    X = layers.Dense(dense_layer_sizes[0], name = 'dense0')(X)\n",
    "    X = layers.Concatenate()([meta_out, X])\n",
    "    \n",
    "    X = layers.BatchNormalization(name = 'bn0')(X)\n",
    "    X = layers.Activation('relu')(X)\n",
    "    X = layers.Dropout(dropout_rate, seed = 7)(X)\n",
    "\n",
    "    # Output layer\n",
    "    X = layers.Dense(3, name = 'output', kernel_regularizer = regularizers.l2(lambd))(X)\n",
    "    X = layers.Activation('softmax')(X)\n",
    "\n",
    "    # Create model\n",
    "    model = models.Model(input = [X_input, meta_input], output = X, name = \"classif_model\")\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold, model_tag,\n",
    "                dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "                oof_folder='oof/', pred_folder='outputs/'):\n",
    "\n",
    "    tag = tag + str(layer) # follow the original naming style\n",
    "    dev_filename = tag + dev_filename\n",
    "    val_filename = tag + val_filename\n",
    "    test_filename = tag + test_filename\n",
    "    \n",
    "    development = pd.read_json(os.path.join(dev_folder_path, dev_filename))\n",
    "    X_development, Y_development = parse_json(development, embedding_size)\n",
    "\n",
    "    remove_development = [row for row in range(len(X_development)) if np.sum(np.isnan(X_development[row].reshape(-1)))]\n",
    "    X_development[remove_development] = np.zeros((3 * embedding_size))\n",
    "\n",
    "    # Will predict probabilities for data from the gap-development file; initializing the predictions\n",
    "    prediction = np.zeros((len(X_development), 3)) # testing predictions\n",
    "    \n",
    "    for fold_n in range(n_fold):\n",
    "        # split training and validation data\n",
    "        print('Make predictions for', fold_n, 'started at', time.ctime())\n",
    "        # Define the model, re-initializing for each fold\n",
    "        classif_model = build_mlp_model([X_development.shape[-1]], split_size=embedding_size, meta_features_shape=[meta_features_test.shape[-1]])\n",
    "        classif_model.load_weights(os.path.join(checkpoint_path, model_tag + tag + str(fold_n) + '.pt'))\n",
    "\n",
    "        pred = classif_model.predict(x=[X_development, meta_features_test], verbose=0)\n",
    "        prediction += pred\n",
    "    \n",
    "    prediction /= n_fold\n",
    "    \n",
    "    submission = pd.read_csv(\"data/sample_submission_stage_2.csv\", index_col = \"ID\")\n",
    "    submission[\"A\"] = prediction[:,0]\n",
    "    submission[\"B\"] = prediction[:,1]\n",
    "    submission[\"NEITHER\"] = prediction[:,2]\n",
    "    submission.to_csv(pred_folder + model_tag + tag + \".csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 22:17:52 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 22:18:01 2019\n",
      "Make predictions for 2 started at Fri Apr 19 22:18:09 2019\n",
      "Make predictions for 3 started at Fri Apr 19 22:18:18 2019\n",
      "Make predictions for 4 started at Fri Apr 19 22:18:26 2019\n",
      "Make predictions for 5 started at Fri Apr 19 22:18:35 2019\n",
      "Make predictions for 6 started at Fri Apr 19 22:18:43 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_base\"\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-base-uncased-seq512-\"\n",
    "pred_tag = \"Hnaive-\"\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 22:18:56 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 22:19:06 2019\n",
      "Make predictions for 2 started at Fri Apr 19 22:19:15 2019\n",
      "Make predictions for 3 started at Fri Apr 19 22:19:24 2019\n",
      "Make predictions for 4 started at Fri Apr 19 22:19:33 2019\n",
      "Make predictions for 5 started at Fri Apr 19 22:19:42 2019\n",
      "Make predictions for 6 started at Fri Apr 19 22:19:51 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_base_cased\"\n",
    "embedding_size = 768\n",
    "layer = 8\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-base-cased-seq512-\"\n",
    "pred_tag = \"Hnaive-\"\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 22:20:05 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 22:20:14 2019\n",
      "Make predictions for 2 started at Fri Apr 19 22:20:23 2019\n",
      "Make predictions for 3 started at Fri Apr 19 22:20:33 2019\n",
      "Make predictions for 4 started at Fri Apr 19 22:20:42 2019\n",
      "Make predictions for 5 started at Fri Apr 19 22:20:51 2019\n",
      "Make predictions for 6 started at Fri Apr 19 22:21:00 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_big\"\n",
    "embedding_size = 1024\n",
    "layer = 19\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-large-uncased-seq300-\"\n",
    "pred_tag = \"Hnaive-\"\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 0 started at Fri Apr 19 22:21:15 2019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zake7\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: UserWarning: Update your `Model` call to the Keras 2 API: `Model(name=\"classif_model\", inputs=[<tf.Tenso..., outputs=Tensor(\"ac...)`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Make predictions for 1 started at Fri Apr 19 22:21:25 2019\n",
      "Make predictions for 2 started at Fri Apr 19 22:21:34 2019\n",
      "Make predictions for 3 started at Fri Apr 19 22:21:43 2019\n",
      "Make predictions for 4 started at Fri Apr 19 22:21:52 2019\n",
      "Make predictions for 5 started at Fri Apr 19 22:22:01 2019\n",
      "Make predictions for 6 started at Fri Apr 19 22:22:10 2019\n"
     ]
    }
   ],
   "source": [
    "dev_folder_path = val_folder_path = test_folder_path = \"vector/bert_big_cased\"\n",
    "embedding_size = 1024\n",
    "layer = 18\n",
    "checkpoint_path = \"stage_1_checkpoints/\"\n",
    "n_fold = 7\n",
    "tag = \"bert-large-cased-seq300-\"\n",
    "pred_tag = \"Hnaive-\"\n",
    "\n",
    "make_predictions(tag, dev_folder_path, val_folder_path, test_folder_path, embedding_size, layer, checkpoint_path, n_fold,\n",
    "            dev_filename=dev_filename, val_filename=val_filename, test_filename=test_filename,\n",
    "            oof_folder='oof/', pred_folder='test_outputs/', model_tag=pred_tag)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Predictons for stage 1.42"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load the predictions of base models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bnli-mh-bert-base-cased-seq512-8.csv',\n",
       " 'bnli-mh-bert-base-uncased-seq512-8.csv',\n",
       " 'bnli-mh-bert-large-cased-seq300-18.csv',\n",
       " 'bnli-mh-bert-large-uncased-seq300-19.csv',\n",
       " 'HBnli-mh-bert-base-cased-seq512-8.csv',\n",
       " 'HBnli-mh-bert-base-uncased-seq512-8.csv',\n",
       " 'HBnli-mh-bert-large-cased-seq300-18.csv',\n",
       " 'HBnli-mh-bert-large-uncased-seq300-19.csv',\n",
       " 'Hnaive-bert-base-cased-seq512-8.csv',\n",
       " 'Hnaive-bert-base-uncased-seq512-8.csv',\n",
       " 'Hnaive-bert-large-cased-seq300-18.csv',\n",
       " 'Hnaive-bert-large-uncased-seq300-19.csv',\n",
       " 'Hnli-mh-bert-base-cased-seq512-8.csv',\n",
       " 'Hnli-mh-bert-base-uncased-seq512-8.csv',\n",
       " 'Hnli-mh-bert-large-cased-seq300-18.csv',\n",
       " 'Hnli-mh-bert-large-uncased-seq300-19.csv',\n",
       " 'naive-bert-base-cased-seq512-8.csv',\n",
       " 'naive-bert-base-uncased-seq512-8.csv',\n",
       " 'naive-bert-large-cased-seq300-18.csv',\n",
       " 'naive-bert-large-uncased-seq300-19.csv',\n",
       " 'nli-mh-bert-base-cased-seq512-8.csv',\n",
       " 'nli-mh-bert-base-uncased-seq512-8.csv',\n",
       " 'nli-mh-bert-large-cased-seq300-18.csv',\n",
       " 'nli-mh-bert-large-uncased-seq300-19.csv']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir('test_outputs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "preds shape (12359, 72)\n"
     ]
    }
   ],
   "source": [
    "preds = [pd.read_csv(os.path.join('test_outputs', pred_file)).values[:, 1:] for pred_file in pred_files]\n",
    "test_x = np.concatenate(preds, axis=1)\n",
    "print(\"preds shape\", test_x.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.models import Model\n",
    "from keras.layers import *\n",
    "from sklearn.model_selection import KFold\n",
    "from keras.optimizers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_mlp_model(feature_nums):\n",
    "    features_inputs = Input(shape=feature_nums, name='mata-features', dtype=\"float32\")\n",
    "    features = features_inputs\n",
    "    \n",
    "    depth = 5\n",
    "    for i in range(depth):\n",
    "        new_features = Dense(24, activation='relu')(features)\n",
    "        new_features = Dropout(0.5)(new_features)\n",
    "        features = Concatenate()([features, new_features])\n",
    "\n",
    "    out_ = Dense(3, activation='softmax')(features)\n",
    "    \n",
    "    model = Model(inputs=[features_inputs], outputs=out_)\n",
    "    model.compile(optimizer=Adam(lr=1e-3, decay=1e-6,), loss='categorical_crossentropy',\n",
    "    metrics=['accuracy',])\n",
    "    model.summary()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 72)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1233 (Dense)              (None, 24)           1752        mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1657 (Dropout)          (None, 24)           0           dense_1233[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1275 (Concatenate)  (None, 96)           0           mata-features[0][0]              \n",
      "                                                                 dropout_1657[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1234 (Dense)              (None, 24)           2328        concatenate_1275[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1658 (Dropout)          (None, 24)           0           dense_1234[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1276 (Concatenate)  (None, 120)          0           concatenate_1275[0][0]           \n",
      "                                                                 dropout_1658[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1235 (Dense)              (None, 24)           2904        concatenate_1276[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1659 (Dropout)          (None, 24)           0           dense_1235[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1277 (Concatenate)  (None, 144)          0           concatenate_1276[0][0]           \n",
      "                                                                 dropout_1659[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1236 (Dense)              (None, 24)           3480        concatenate_1277[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1660 (Dropout)          (None, 24)           0           dense_1236[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1278 (Concatenate)  (None, 168)          0           concatenate_1277[0][0]           \n",
      "                                                                 dropout_1660[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1237 (Dense)              (None, 24)           4056        concatenate_1278[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1661 (Dropout)          (None, 24)           0           dense_1237[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1279 (Concatenate)  (None, 192)          0           concatenate_1278[0][0]           \n",
      "                                                                 dropout_1661[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1238 (Dense)              (None, 3)            579         concatenate_1279[0][0]           \n",
      "==================================================================================================\n",
      "Total params: 15,099\n",
      "Trainable params: 15,099\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 72)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1239 (Dense)              (None, 24)           1752        mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1662 (Dropout)          (None, 24)           0           dense_1239[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1280 (Concatenate)  (None, 96)           0           mata-features[0][0]              \n",
      "                                                                 dropout_1662[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1240 (Dense)              (None, 24)           2328        concatenate_1280[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1663 (Dropout)          (None, 24)           0           dense_1240[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1281 (Concatenate)  (None, 120)          0           concatenate_1280[0][0]           \n",
      "                                                                 dropout_1663[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1241 (Dense)              (None, 24)           2904        concatenate_1281[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1664 (Dropout)          (None, 24)           0           dense_1241[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1282 (Concatenate)  (None, 144)          0           concatenate_1281[0][0]           \n",
      "                                                                 dropout_1664[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1242 (Dense)              (None, 24)           3480        concatenate_1282[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1665 (Dropout)          (None, 24)           0           dense_1242[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1283 (Concatenate)  (None, 168)          0           concatenate_1282[0][0]           \n",
      "                                                                 dropout_1665[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1243 (Dense)              (None, 24)           4056        concatenate_1283[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1666 (Dropout)          (None, 24)           0           dense_1243[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1284 (Concatenate)  (None, 192)          0           concatenate_1283[0][0]           \n",
      "                                                                 dropout_1666[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1244 (Dense)              (None, 3)            579         concatenate_1284[0][0]           \n",
      "==================================================================================================\n",
      "Total params: 15,099\n",
      "Trainable params: 15,099\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 72)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1245 (Dense)              (None, 24)           1752        mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1667 (Dropout)          (None, 24)           0           dense_1245[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1285 (Concatenate)  (None, 96)           0           mata-features[0][0]              \n",
      "                                                                 dropout_1667[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1246 (Dense)              (None, 24)           2328        concatenate_1285[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1668 (Dropout)          (None, 24)           0           dense_1246[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1286 (Concatenate)  (None, 120)          0           concatenate_1285[0][0]           \n",
      "                                                                 dropout_1668[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1247 (Dense)              (None, 24)           2904        concatenate_1286[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1669 (Dropout)          (None, 24)           0           dense_1247[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1287 (Concatenate)  (None, 144)          0           concatenate_1286[0][0]           \n",
      "                                                                 dropout_1669[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1248 (Dense)              (None, 24)           3480        concatenate_1287[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1670 (Dropout)          (None, 24)           0           dense_1248[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1288 (Concatenate)  (None, 168)          0           concatenate_1287[0][0]           \n",
      "                                                                 dropout_1670[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1249 (Dense)              (None, 24)           4056        concatenate_1288[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1671 (Dropout)          (None, 24)           0           dense_1249[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1289 (Concatenate)  (None, 192)          0           concatenate_1288[0][0]           \n",
      "                                                                 dropout_1671[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1250 (Dense)              (None, 3)            579         concatenate_1289[0][0]           \n",
      "==================================================================================================\n",
      "Total params: 15,099\n",
      "Trainable params: 15,099\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 72)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1251 (Dense)              (None, 24)           1752        mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1672 (Dropout)          (None, 24)           0           dense_1251[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1290 (Concatenate)  (None, 96)           0           mata-features[0][0]              \n",
      "                                                                 dropout_1672[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1252 (Dense)              (None, 24)           2328        concatenate_1290[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1673 (Dropout)          (None, 24)           0           dense_1252[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1291 (Concatenate)  (None, 120)          0           concatenate_1290[0][0]           \n",
      "                                                                 dropout_1673[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1253 (Dense)              (None, 24)           2904        concatenate_1291[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1674 (Dropout)          (None, 24)           0           dense_1253[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1292 (Concatenate)  (None, 144)          0           concatenate_1291[0][0]           \n",
      "                                                                 dropout_1674[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1254 (Dense)              (None, 24)           3480        concatenate_1292[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1675 (Dropout)          (None, 24)           0           dense_1254[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1293 (Concatenate)  (None, 168)          0           concatenate_1292[0][0]           \n",
      "                                                                 dropout_1675[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1255 (Dense)              (None, 24)           4056        concatenate_1293[0][0]           \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "dropout_1676 (Dropout)          (None, 24)           0           dense_1255[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1294 (Concatenate)  (None, 192)          0           concatenate_1293[0][0]           \n",
      "                                                                 dropout_1676[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1256 (Dense)              (None, 3)            579         concatenate_1294[0][0]           \n",
      "==================================================================================================\n",
      "Total params: 15,099\n",
      "Trainable params: 15,099\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 72)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1257 (Dense)              (None, 24)           1752        mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1677 (Dropout)          (None, 24)           0           dense_1257[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1295 (Concatenate)  (None, 96)           0           mata-features[0][0]              \n",
      "                                                                 dropout_1677[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1258 (Dense)              (None, 24)           2328        concatenate_1295[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1678 (Dropout)          (None, 24)           0           dense_1258[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1296 (Concatenate)  (None, 120)          0           concatenate_1295[0][0]           \n",
      "                                                                 dropout_1678[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1259 (Dense)              (None, 24)           2904        concatenate_1296[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1679 (Dropout)          (None, 24)           0           dense_1259[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1297 (Concatenate)  (None, 144)          0           concatenate_1296[0][0]           \n",
      "                                                                 dropout_1679[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1260 (Dense)              (None, 24)           3480        concatenate_1297[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1680 (Dropout)          (None, 24)           0           dense_1260[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1298 (Concatenate)  (None, 168)          0           concatenate_1297[0][0]           \n",
      "                                                                 dropout_1680[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1261 (Dense)              (None, 24)           4056        concatenate_1298[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1681 (Dropout)          (None, 24)           0           dense_1261[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1299 (Concatenate)  (None, 192)          0           concatenate_1298[0][0]           \n",
      "                                                                 dropout_1681[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1262 (Dense)              (None, 3)            579         concatenate_1299[0][0]           \n",
      "==================================================================================================\n",
      "Total params: 15,099\n",
      "Trainable params: 15,099\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 72)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1263 (Dense)              (None, 24)           1752        mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1682 (Dropout)          (None, 24)           0           dense_1263[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1300 (Concatenate)  (None, 96)           0           mata-features[0][0]              \n",
      "                                                                 dropout_1682[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1264 (Dense)              (None, 24)           2328        concatenate_1300[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1683 (Dropout)          (None, 24)           0           dense_1264[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1301 (Concatenate)  (None, 120)          0           concatenate_1300[0][0]           \n",
      "                                                                 dropout_1683[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1265 (Dense)              (None, 24)           2904        concatenate_1301[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1684 (Dropout)          (None, 24)           0           dense_1265[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1302 (Concatenate)  (None, 144)          0           concatenate_1301[0][0]           \n",
      "                                                                 dropout_1684[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1266 (Dense)              (None, 24)           3480        concatenate_1302[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1685 (Dropout)          (None, 24)           0           dense_1266[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1303 (Concatenate)  (None, 168)          0           concatenate_1302[0][0]           \n",
      "                                                                 dropout_1685[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1267 (Dense)              (None, 24)           4056        concatenate_1303[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1686 (Dropout)          (None, 24)           0           dense_1267[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1304 (Concatenate)  (None, 192)          0           concatenate_1303[0][0]           \n",
      "                                                                 dropout_1686[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1268 (Dense)              (None, 3)            579         concatenate_1304[0][0]           \n",
      "==================================================================================================\n",
      "Total params: 15,099\n",
      "Trainable params: 15,099\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "mata-features (InputLayer)      (None, 72)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1269 (Dense)              (None, 24)           1752        mata-features[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1687 (Dropout)          (None, 24)           0           dense_1269[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1305 (Concatenate)  (None, 96)           0           mata-features[0][0]              \n",
      "                                                                 dropout_1687[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1270 (Dense)              (None, 24)           2328        concatenate_1305[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1688 (Dropout)          (None, 24)           0           dense_1270[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1306 (Concatenate)  (None, 120)          0           concatenate_1305[0][0]           \n",
      "                                                                 dropout_1688[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1271 (Dense)              (None, 24)           2904        concatenate_1306[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1689 (Dropout)          (None, 24)           0           dense_1271[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1307 (Concatenate)  (None, 144)          0           concatenate_1306[0][0]           \n",
      "                                                                 dropout_1689[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1272 (Dense)              (None, 24)           3480        concatenate_1307[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1690 (Dropout)          (None, 24)           0           dense_1272[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1308 (Concatenate)  (None, 168)          0           concatenate_1307[0][0]           \n",
      "                                                                 dropout_1690[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1273 (Dense)              (None, 24)           4056        concatenate_1308[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1691 (Dropout)          (None, 24)           0           dense_1273[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1309 (Concatenate)  (None, 192)          0           concatenate_1308[0][0]           \n",
      "                                                                 dropout_1691[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "dense_1274 (Dense)              (None, 3)            579         concatenate_1309[0][0]           \n",
      "==================================================================================================\n",
      "Total params: 15,099\n",
      "Trainable params: 15,099\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "prediction = np.zeros((len(test_x), 3)) # testing predictions\n",
    "n_fold = 7\n",
    "for fold_n in range(n_fold):\n",
    "    classif_model = build_mlp_model([test_x.shape[-1]])\n",
    "    classif_model.load_weights('stage_1_finals/nn_ensemble_checkpoint_f{}.pt'.format(fold_n))\n",
    "    pred = classif_model.predict(x=test_x, verbose=0)\n",
    "    prediction += pred\n",
    "    \n",
    "prediction /= n_fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# make a submission\n",
    "submission = pd.read_csv(\"data/sample_submission_stage_2.csv\", index_col = \"ID\")\n",
    "submission[\"A\"] = prediction[:,0]\n",
    "submission[\"B\"] = prediction[:,1]\n",
    "submission[\"NEITHER\"] = prediction[:,2]\n",
    "submission.to_csv(\"stage_2_finals/s3_24_nn_ensembles.csv\")    "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}