Step3_pure_bert_model.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Step3-pure_bert_model.ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[],"toc_visible":true},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"metadata":{"id":"jqptSekdUXln","colab_type":"text"},"cell_type":"markdown","source":["**end2end model: Using dev, test, val jsons from step1, train pure bert model for both sub_A and sub_B and save 50 weights files each **\n","\n","sub_A: using 2000 development + 2000 test + 400 validation to train\n","\n","sub_B: using 2000 test + 454 validation to train\n","\n","** Instruction:**\n","\n","run the notebook 4 times, with `all_train = True` (sub_A) and` False` (sub_B) and `CASED = False` and `True` respectively"]},{"metadata":{"id":"JD8GTddpc-38","colab_type":"text"},"cell_type":"markdown","source":["## set up"]},{"metadata":{"id":"qww4nxqTrZV4","colab_type":"code","colab":{}},"cell_type":"code","source":["path = 'drive/My Drive/pronoun/'"],"execution_count":0,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true,"id":"FO108906rI8M","colab_type":"code","colab":{}},"cell_type":"code","source":["import numpy as np # linear algebra\n","import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n","import os\n","import zipfile\n","import gc\n","from tqdm import tqdm as tqdm\n","import re\n","from glob import glob\n","import shutil\n","\n","from sklearn.model_selection import KFold\n","from sklearn.metrics import log_loss"],"execution_count":0,"outputs":[]},{"metadata":{"_uuid":"e265e7becd34d793d13009f851a4fc7c6f7f95fa","trusted":true,"id":"nPXSW1XBrI8h","colab_type":"code","colab":{}},"cell_type":"code","source":["!wget https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-development.tsv\n","!wget https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-validation.tsv\n","!wget https://raw.githubusercontent.com/google-research-datasets/gap-coreference/master/gap-test.tsv"],"execution_count":0,"outputs":[]},{"metadata":{"id":"-n9cXFSBlL43","colab_type":"code","outputId":"6cac61ad-0e94-4f20-f57a-edf43fa46ac4","executionInfo":{"status":"ok","timestamp":1554945067781,"user_tz":240,"elapsed":147393,"user":{"displayName":"Bo Liu","photoUrl":"https://lh5.googleusercontent.com/-yAtjaCoYe44/AAAAAAAAAAI/AAAAAAAAAE8/Jc2yMe60vOc/s64/photo.jpg","userId":"00951788254292121969"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["# move json files from Drive to local for fast read (only needed for Colab)\n","for x in tqdm([x for x in sorted(glob('drive/My Drive/pronoun/output/contextual_embeddings_gap*_fix_long_text.json')) if 'LARGE' in x]):\n","  shutil.copy2(x, os.path.basename(x)) "],"execution_count":0,"outputs":[{"output_type":"stream","text":["100%|██████████| 100/100 [02:19<00:00,  1.32s/it]\n"],"name":"stderr"}]},{"metadata":{"id":"eW-yR4O3x0HC","colab_type":"code","colab":{}},"cell_type":"code","source":["all_train = False\n","CASED = False"],"execution_count":0,"outputs":[]},{"metadata":{"id":"NY-otVWUgFAt","colab_type":"text"},"cell_type":"markdown","source":["## make data (np array) from saved json features"]},{"metadata":{"id":"E22hScEWQ42N","colab_type":"code","colab":{}},"cell_type":"code","source":["def parse_json(embeddings, \n","               debug=False,\n","               overwrite_dev_labels = None, # 0 first half, 1 second half\n","               overwrite_test_labels = None, # 0 first half, 1 second half\n","               overwrite_val_labels = False\n","              ):\n","\t'''\n","\tParses the embeddigns given by BERT, and suitably formats them to be passed to the MLP model\n","\n","\tInput: embeddings, a DataFrame containing contextual embeddings from BERT, as well as the labels for the classification problem\n","\tcolumns: \"emb_A\": contextual embedding for the word A\n","\t         \"emb_B\": contextual embedding for the word B\n","\t         \"emb_P\": contextual embedding for the pronoun\n","\t         \"label\": the answer to the coreference problem: \"A\", \"B\" or \"NEITHER\"\n","\n","\tOutput: X, a numpy array containing, for each line in the GAP file, the concatenation of the embeddings of the target words\n","\t        Y, a numpy array containing, for each line in the GAP file, the one-hot encoded answer to the coreference problem\n","\t'''\n","\tembeddings.sort_index(inplace = True) # Sorting the DataFrame, because reading from the json file messed with the order\n","\tnum_token = 3\n","\tBS = 768 if not LARGE else 1024\n","\tX = np.zeros((len(embeddings),num_token*BS)) \n","\tY = np.zeros((len(embeddings), 3))\n","\n","\t# Concatenate features\n","\tfor i in range(len(embeddings)):\n","\t\tA = np.array(embeddings.loc[i,\"emb_A\"])\n","\t\tB = np.array(embeddings.loc[i,\"emb_B\"])\n","\t\tP = np.array(embeddings.loc[i,\"emb_P\"])\n","\t\tX[i] = np.concatenate((A,B,P))\n","\n","\t# One-hot encoding for labels\n","\tfor i in range(len(embeddings)):\n","\t\tlabel = embeddings.loc[i,\"label\"]\n","\t\tif label == \"A\":  Y[i,:] = [1,0,0]\n","\t\telif label == \"B\":Y[i,:] = [0,1,0]\n","\t\telse:             Y[i,:] = [0,0,1]\n","      \n","\tif overwrite_dev_labels==0:\n","\t\tprint('============ USING gap-development-corrected-74 LABELS !!! ============')\n","\t\tassert len(embeddings) == 1000\n","\t\tcnt = 0\n","\t\tfor i in range(1000):\n","\t\t\tdf = pd.read_csv('drive/My Drive/pronoun/input/gap-development-corrected-74.tsv',sep='\\t')\n","\t\t\tlabel = 'A' if df.loc[i,\"A-coref\"] else ('B' if df.loc[i,'B-coref'] else 'Neither')\n","\t\t\tif label!=embeddings.loc[i,\"label\"]: cnt +=1\n","\t\t\tif label == \"A\":  Y[i,:] = [1,0,0]\n","\t\t\telif label == \"B\":Y[i,:] = [0,1,0]\n","\t\t\telse:             Y[i,:] = [0,0,1]  \n","\t\tprint('corrected {:d} labels'.format(cnt))        \n","\telif overwrite_dev_labels==1:\n","\t\tprint('============ USING gap-development-corrected-74 LABELS !!! ============')    \n","\t\tassert len(embeddings) == 1000\n","\t\tcnt = 0    \n","\t\tfor i in range(1000):\n","\t\t\tdf = pd.read_csv('drive/My Drive/pronoun/input/gap-development-corrected-74.tsv',sep='\\t')\n","\t\t\tlabel = 'A' if df.loc[1000+i,\"A-coref\"] else ('B' if df.loc[1000+i,'B-coref'] else 'Neither')\n","\t\t\tif label!=embeddings.loc[i,\"label\"]: cnt +=1      \n","\t\t\tif label == \"A\":  Y[i,:] = [1,0,0]\n","\t\t\telif label == \"B\":Y[i,:] = [0,1,0]\n","\t\t\telse:             Y[i,:] = [0,0,1]          \n","\t\tprint('corrected {:d} labels'.format(cnt))        \n","    \n","\tif overwrite_test_labels==0:\n","\t\tprint('============ USING gap-test-val-85 LABELS (test0) !!! ============')\n","\t\tassert len(embeddings) == 1000\n","\t\tcnt = 0\n","\t\tfor i in range(1000):\n","\t\t\tdf = pd.read_csv('drive/My Drive/pronoun/input/gap-test-val-85.tsv',sep='\\t')\n","\t\t\tlabel = 'A' if df.loc[i,\"A-coref\"] else ('B' if df.loc[i,'B-coref'] else 'Neither')\n","\t\t\tif label!=embeddings.loc[i,\"label\"]: cnt +=1\n","\t\t\tif label == \"A\":  Y[i,:] = [1,0,0]\n","\t\t\telif label == \"B\":Y[i,:] = [0,1,0]\n","\t\t\telse:             Y[i,:] = [0,0,1]  \n","\t\tprint('corrected {:d} labels'.format(cnt))        \n","\telif overwrite_test_labels==1:\n","\t\tprint('============ USING gap-test-val-85 LABELS (test1) !!! ============')    \n","\t\tassert len(embeddings) == 1000\n","\t\tcnt = 0    \n","\t\tfor i in range(1000):\n","\t\t\tdf = pd.read_csv('drive/My Drive/pronoun/input/gap-test-val-85.tsv',sep='\\t')\n","\t\t\tlabel = 'A' if df.loc[1000+i,\"A-coref\"] else ('B' if df.loc[1000+i,'B-coref'] else 'Neither')\n","\t\t\tif label!=embeddings.loc[i,\"label\"]: cnt +=1      \n","\t\t\tif label == \"A\":  Y[i,:] = [1,0,0]\n","\t\t\telif label == \"B\":Y[i,:] = [0,1,0]\n","\t\t\telse:             Y[i,:] = [0,0,1]             \n","\t\tprint('corrected {:d} labels'.format(cnt))    \n","    \n","\tif overwrite_val_labels:\n","\t\tprint('============ USING gap-test-val-85 LABELS (val) !!! ============')\n","\t\tassert len(embeddings) == 454\n","\t\tcnt = 0\n","\t\tfor i in range(454):\n","\t\t\tdf = pd.read_csv('drive/My Drive/pronoun/input/gap-test-val-85.tsv',sep='\\t')\n","\t\t\tlabel = 'A' if df.loc[2000+i,\"A-coref\"] else ('B' if df.loc[2000+i,'B-coref'] else 'Neither')\n","\t\t\tif label!=embeddings.loc[i,\"label\"]: cnt +=1\n","\t\t\tif label == \"A\":  Y[i,:] = [1,0,0]\n","\t\t\telif label == \"B\":Y[i,:] = [0,1,0]\n","\t\t\telse:             Y[i,:] = [0,0,1]       \n","\t\tprint('corrected {:d} labels'.format(cnt))      \n","    \n","\treturn X, Y"],"execution_count":0,"outputs":[]},{"metadata":{"id":"Kt3DUqHdjE-8","colab_type":"code","colab":{}},"cell_type":"code","source":["def get_json_names(CASED = True,\n","                   LARGE = True,\n","                   MAX_SEQ_LEN = 256,\n","                   layer = None,\n","                   concat_lst = [\"-3\",\"-4\"],\n","                   TTA_suffix = ''):\n","  \n","  suffix = layer\n","  if CASED: suffix += '_CASED'\n","  if LARGE: suffix += '_LARGE'    \n","    \n","  suffix += ('_'+TTA_suffix) if TTA_suffix!='' else ''\n","  \n","  json_suffix = '_fix_long_text.json'\n","  \n","  if LARGE:\n","    json_names = ['contextual_embeddings_gap_development_'+suffix+'_1'+json_suffix,\n","                  'contextual_embeddings_gap_development_'+suffix+'_2'+json_suffix,\n","                  'contextual_embeddings_gap_test_'+suffix+'_1'+json_suffix,\n","                  'contextual_embeddings_gap_test_'+suffix+'_2'+json_suffix,\n","                  'contextual_embeddings_gap_validation_'+suffix+json_suffix]            \n","  else:\n","    raise Exception('Not implemented')\n","    \n","  print(json_names)\n","  return json_names\n","\n","\n","def make_np_features_from_json(CASED = True,\n","                               LARGE = True,\n","                               MAX_SEQ_LEN = 256,\n","                               layer = None,\n","                               concat_lst = [\"-3\",\"-4\"],\n","                               TTA_suffix = '',\n","                               all_train = False):  \n","  # single layer\n","  if concat_lst == None:\n","    json_names = get_json_names(CASED, LARGE, MAX_SEQ_LEN, layer, concat_lst, TTA_suffix)\n","    validation = pd.read_json(json_names[-1])\n","    X_validation, Y_validation = parse_json(validation, overwrite_val_labels = True)\n","\n","    development = pd.read_json(json_names[0])\n","    X_development1, Y_development1 = parse_json(development, overwrite_dev_labels=0 if all_train else None)\n","    development = pd.read_json(json_names[1])\n","    X_development2, Y_development2 = parse_json(development, overwrite_dev_labels=1 if all_train else None)\n","\n","    X_development = np.concatenate((X_development1,X_development2))\n","    Y_development = np.concatenate((Y_development1,Y_development2))\n","\n","    test = pd.read_json(json_names[2])\n","    X_test1, Y_test1 = parse_json(test, overwrite_test_labels = 0)\n","    test = pd.read_json(json_names[3])\n","    X_test2, Y_test2 = parse_json(test, overwrite_test_labels = 1)\n","    X_test = np.concatenate((X_test1,X_test2))\n","    Y_test = np.concatenate((Y_test1,Y_test2))    \n","      \n","    ## remove NaN rows, and combine train data\n","  \n","    if all_train:\n","      # train: 4400\n","      # sanity: 454\n","      np.random.seed(15)\n","      sanity_idx = np.random.choice(454,54,replace=False)\n","      val_idx = np.setdiff1d(np.arange(454),sanity_idx)     \n","      \n","      X_train = np.concatenate((X_development, X_test, X_validation[val_idx,:]), axis = 0).copy()\n","      Y_train = np.concatenate((Y_development, Y_test, Y_validation[val_idx,:]), axis = 0).copy()\n","      X_development = X_validation[sanity_idx,:]\n","      Y_development = Y_validation[sanity_idx,:]\n","      \n","      remove_development = [row for row in range(len(X_development)) if np.sum(np.isnan(X_development[row]))]\n","      print('remove_development: ' + str(remove_development))\n","      num_token = 3\n","      num_concat = len(concat_lst) if concat_lst != None else 1\n","      BS = 1024 if LARGE else 768\n","      X_development[remove_development] = np.zeros(num_token*BS*num_concat)            \n","\n","    else:\n","      # train: test 2000 + val 454\n","      # predict: dev 2000\n","      # We want predictions for all development rows. So instead of removing rows, make them 0\n","      remove_development = [row for row in range(len(X_development)) if np.sum(np.isnan(X_development[row]))]\n","      print('remove_development: ' + str(remove_development))\n","      num_token = 3\n","      num_concat = len(concat_lst) if concat_lst != None else 1\n","      BS = 1024 if LARGE else 768\n","      X_development[remove_development] = np.zeros(num_token*BS*num_concat)\n","\n","      # Will train on data from the gap-test and gap-validation files, in total 2454 rows\n","      X_train = np.concatenate((X_test, X_validation), axis = 0)\n","      Y_train = np.concatenate((Y_test, Y_validation), axis = 0)       \n","         \n","  # concat, recursive\n","  else:   \n","    for this_layer in concat_lst:      \n","      # recursive\n","      X_train_layer, Y_train_layer, X_development_layer, Y_development_layer = \\\n","          make_np_features_from_json(CASED, LARGE, MAX_SEQ_LEN, this_layer, None, TTA_suffix, all_train)\n","\n","      if this_layer==concat_lst[0]:\n","        X_development, Y_development = X_development_layer, Y_development_layer\n","        X_train, Y_train = X_train_layer, Y_train_layer\n","      else:\n","        X_development = np.concatenate((X_development,X_development_layer),axis=1)\n","        X_train = np.concatenate((X_train,X_train_layer),axis=1)  \n","    \n","  return X_train, Y_train, X_development, Y_development       \n","        "],"execution_count":0,"outputs":[]},{"metadata":{"id":"TcbhU8ikeWY9","colab_type":"code","colab":{}},"cell_type":"code","source":["%%time\n","\n","LARGE = True\n","concat_lst = [\"-3\",\"-4\"]\n","layer = None # \"-3\"\n","MAX_SEQ_LEN = 256\n","\n","TTA_suffixes = [\\\n","                 'Alice_Kate_John_Michael',\n","                 'Elizabeth_Mary_James_Henry',\n","                 'Kate_Elizabeth_Michael_James',\n","                 'Mary_Alice_Henry_John']\n","\n","d_XY = {}\n","\n","for TTA_suffix in ['orig'] + TTA_suffixes:\n","  this_d = {}\n","  \n","  this_d['X_train'],this_d['Y_train'],this_d['X_dev'],this_d['Y_dev'] = \\\n","          make_np_features_from_json(CASED = CASED,\n","                                     LARGE = LARGE,\n","                                     MAX_SEQ_LEN = MAX_SEQ_LEN,\n","                                     layer = layer,\n","                                     concat_lst = concat_lst,\n","                                     TTA_suffix = '' if TTA_suffix=='orig' else TTA_suffix,\n","                                     all_train=all_train) \n","  print(this_d['X_train'].shape, this_d['Y_train'].shape, this_d['X_dev'].shape, this_d['Y_dev'].shape)\n","  \n","  d_XY[TTA_suffix] = this_d\n","\n","print(d_XY['orig']['X_train'].shape, d_XY['orig']['Y_train'].shape, d_XY['orig']['X_dev'].shape, d_XY['orig']['Y_dev'].shape)  \n"],"execution_count":0,"outputs":[]},{"metadata":{"id":"WoQWnV4WQ4Wu","colab_type":"code","outputId":"915f4853-7fc0-4f15-b9a8-ea0457a65d6e","executionInfo":{"status":"ok","timestamp":1554945817202,"user_tz":240,"elapsed":1229,"user":{"displayName":"Bo Liu","photoUrl":"https://lh5.googleusercontent.com/-yAtjaCoYe44/AAAAAAAAAAI/AAAAAAAAAE8/Jc2yMe60vOc/s64/photo.jpg","userId":"00951788254292121969"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["# remove missing rows in both original and each Aug\n","\n","index_train = list(range(d_XY['orig']['X_train'].shape[0]))\n","remove_train = [row for row in range(len(d_XY['orig']['X_train'])) if np.sum(np.isnan(d_XY['orig']['X_train'][row]))]\n","d_XY['orig']['X_train'] = np.delete(d_XY['orig']['X_train'], remove_train, 0)\n","d_XY['orig']['Y_train'] = np.delete(d_XY['orig']['Y_train'], remove_train, 0)\n","\n","# which rows are left from 0 to 2453\n","index_train = np.delete(index_train, remove_train, 0)\n","\n","print(\"removed: \", remove_train, len(remove_train))  "],"execution_count":0,"outputs":[{"output_type":"stream","text":["removed:  [] 0\n"],"name":"stdout"}]},{"metadata":{"id":"wqtX7IwmtZew","colab_type":"code","colab":{}},"cell_type":"code","source":["for k,v in d_XY.items():\n","  if k=='orig':\n","    continue\n","  remove_train_aug = [row for row in range(len(v['X_train'])) if np.sum(np.isnan(v['X_train'][row]))]\n","  print(k, remove_train_aug)\n","\n","  # remove rows that are also missing in orignal\n","  v['X_train'] = np.delete(v['X_train'], remove_train, 0)\n","  v['Y_train'] = np.delete(v['Y_train'], remove_train, 0)\n","\n","# sanity check\n","print('------- sanity check ---------')\n","for k,v in d_XY.items():\n","  print(k, v['X_train'].shape, v['Y_train'].shape, v['X_dev'].shape, v['Y_dev'].shape)  \n","\n","print(d_XY['orig']['X_train'].shape, d_XY['orig']['Y_train'].shape, d_XY['orig']['X_dev'].shape, d_XY['orig']['Y_dev'].shape)   "],"execution_count":0,"outputs":[]},{"metadata":{"id":"6HUNH6Kt_wDN","colab_type":"text"},"cell_type":"markdown","source":["## keras model"]},{"metadata":{"id":"6YQ9D96fyDJJ","colab_type":"text"},"cell_type":"markdown","source":["the model architecture is based on https://www.kaggle.com/mateiionita/taming-the-bert-a-baseline"]},{"metadata":{"id":"DzCiBk2n_lqf","colab_type":"code","outputId":"c2505199-ead1-457a-896c-aeb11e1b59a0","executionInfo":{"status":"ok","timestamp":1554945879787,"user_tz":240,"elapsed":3369,"user":{"displayName":"Bo Liu","photoUrl":"https://lh5.googleusercontent.com/-yAtjaCoYe44/AAAAAAAAAAI/AAAAAAAAAE8/Jc2yMe60vOc/s64/photo.jpg","userId":"00951788254292121969"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["from keras import backend, models, layers, initializers, regularizers, constraints, optimizers\n","from keras import callbacks as kc\n","from keras import optimizers as ko\n","\n","from sklearn.model_selection import cross_val_score, KFold, train_test_split\n","from sklearn.metrics import log_loss\n","import time\n","\n","\n","def build_mlp_model(input_shape):\n","\tX_input = layers.Input(input_shape)\n","\n","\t# First dense layer\n","\tX = layers.Dense(dense_layer_sizes[0], name = 'dense0')(X_input)\n","\tX = layers.BatchNormalization(name = 'bn0')(X)\n","\tX = layers.Activation('relu')(X)\n","\tX = layers.Dropout(dropout_rate, seed = 7)(X)\n","\n","\t# Second dense layer\n","\tif len(dense_layer_sizes)==2:\n","\t\tX = layers.Dense(dense_layer_sizes[1], name = 'dense1')(X)\n","\t\tX = layers.BatchNormalization(name = 'bn1')(X)\n","\t\tX = layers.Activation('relu')(X)\n","\t\tX = layers.Dropout(dropout_rate, seed = 9)(X)\n","\n","\t# Output layer\n","\tX = layers.Dense(3, name = 'output', kernel_regularizer = regularizers.l2(lambd))(X)\n","\tX = layers.Activation('softmax')(X)\n","\n","\t# Create model\n","\tmodel = models.Model(input = X_input, output = X, name = \"classif_model\")\n","\treturn model"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Using TensorFlow backend.\n"],"name":"stderr"}]},{"metadata":{"id":"7Hp3xaLGAMc4","colab_type":"code","outputId":"52b9c973-ba79-4d93-bd24-cdfb9bbd49b6","executionInfo":{"status":"ok","timestamp":1554946041001,"user_tz":240,"elapsed":2131,"user":{"displayName":"Bo Liu","photoUrl":"https://lh5.googleusercontent.com/-yAtjaCoYe44/AAAAAAAAAAI/AAAAAAAAAE8/Jc2yMe60vOc/s64/photo.jpg","userId":"00951788254292121969"}},"colab":{"base_uri":"https://localhost:8080/","height":52}},"cell_type":"code","source":["loss = \"categorical_crossentropy\"\n","\n","dense_layer_sizes = [64]\n","if concat_lst != None:\n","  dense_layer_sizes = [512,32]\n","dropout_rate = 0.6\n","learning_rate = 0.001\n","n_fold = 5\n","batch_size = 32\n","epochs = 1000\n","patience_orig = 60 if all_train else 100\n","patience = patience_orig\n","lambd = 0.1 # L2 regularization\n","\n","np.random.seed(15)\n","sanity_idx = np.random.choice(454,54,replace=False)\n","val_idx = np.setdiff1d(np.arange(454),sanity_idx)\n","bad_rows = remove_train\n","\n","suffix = 'pure_bert_' + ''.join(concat_lst)\n","if CASED: suffix += '_CASED'\n","if LARGE: suffix += '_LARGE'\n","    \n","TTA_suffixes = \\\n","['orig',\n"," 'Alice_Kate_John_Michael',\n"," 'Elizabeth_Mary_James_Henry',\n"," 'Kate_Elizabeth_Michael_James',\n"," 'Mary_Alice_Henry_John']\n","\n","Aug_suffixes = \\\n","['Alice_Kate_John_Michael',\n","#  'Elizabeth_Mary_James_Henry',\n"," 'Kate_Elizabeth_Michael_James',\n"," 'Mary_Alice_Henry_John']\n","\n","if len(Aug_suffixes)==3: suffix += '_Aug3'\n","if len(Aug_suffixes)==4: suffix += '_Aug4'  \n","suffix += '_all_train' if all_train else '_sub_B'\n","  \n","suffix += '_4400'\n","\n","print(suffix)  \n","\n","num_test = d_XY['orig']['X_dev'].shape[0]\n","num_train = d_XY['orig']['X_train'].shape[0]\n","print(num_train,num_test)"],"execution_count":0,"outputs":[{"output_type":"stream","text":["pure_bert_-3-4_LARGE_Aug3_sub_B_4400\n","2454 2000\n"],"name":"stdout"}]},{"metadata":{"id":"RIRPuCap_yFH","colab_type":"code","colab":{}},"cell_type":"code","source":["%%time\n","\n","#####  all_train\n","# pred_oof, sub_all_d: OOF\n","# pred_all_d, sub_df : sanity\n","\n","#####  sub_B\n","# pred_all_d, sub_df : dev\n","\n","if all_train:\n","\n","  pd.DataFrame(columns=['a','b']).to_csv(path+'sub/tmp.csv') # testing drive connection\n","\n","  for run in range(0,1):\n","    import gc; gc.collect()\n","\n","    sub_all = pd.concat([pd.read_table(path+'input/gap-development.tsv',usecols=['ID']),\n","                         pd.read_table(path+'input/gap-test.tsv',usecols=['ID']),\n","                         pd.read_table(path+'input/gap-validation.tsv',usecols=['ID']).iloc[val_idx]]).\\\n","              reset_index(drop=True).drop(bad_rows)\n","    assert sub_all.shape[0]==num_train\n","    sub_all['A']=0; sub_all['B']=0; sub_all['NEITHER']=0\n","    sub_all_d = {}\n","    for TTA_suffix in TTA_suffixes: sub_all_d[TTA_suffix] = sub_all.copy()  \n","\n","    pred_all_d = {} # to save 25 fold avg (for Test), 5 outer OOF, 5 inner early stop\n","    for TTA_suffix in TTA_suffixes: pred_all_d[TTA_suffix] = np.zeros((num_test,3))        \n","\n","    # outer 5 fold: OOF fold. 4/5 train, 1/5 OOF pred  \n","    kfold = KFold(n_splits=n_fold, shuffle=True, random_state=3)\n","    for fold_n, (train_fold_index, oof_val_index) in enumerate(kfold.split(d_XY['orig']['X_train'])):\n","      y_train_fold= d_XY['orig']['Y_train'][train_fold_index,:]\n","\n","      X_train_fold_d = {}; X_oof_val_d = {}\n","      for TTA_suffix in TTA_suffixes: \n","        X_train_fold_d[TTA_suffix]= d_XY[TTA_suffix]['X_train'][train_fold_index,:]   \n","        X_oof_val_d[TTA_suffix]   = d_XY[TTA_suffix]['X_train'][oof_val_index,:]   \n","\n","      # inner 5 fold: train and early-stop val fold.\n","      kfold_inner = KFold(n_splits=n_fold, shuffle=True, random_state=5)\n","      for fold_n_inner, (train_index, valid_index) in enumerate(kfold_inner.split(X_train_fold_d['orig'])):        \n","\n","        X_tr  = X_train_fold_d['orig'][train_index,:]\n","        X_tr_orig = X_tr.copy()\n","        X_val = X_train_fold_d['orig'][valid_index,:]\n","        y_tr  = y_train_fold[train_index,:]\n","        y_val = y_train_fold[valid_index,:]              \n","\n","        # train augmentation\n","        if len(Aug_suffixes)>1: \n","          patience = np.ceil(patience_orig / (1+len(Aug_suffixes)))\n","\n","          for k in Aug_suffixes:\n","            X_tr = np.concatenate((X_tr, X_train_fold_d[k][train_index,:]),axis=0)\n","            y_tr = np.concatenate((y_tr, y_tr),axis=0)\n","\n","          arr = np.arange(X_tr.shape[0])\n","          np.random.shuffle(arr)\n","          X_tr = X_tr[arr,:] \n","          y_tr = y_tr[arr,:]  \n","          print(X_tr.shape, y_tr.shape, X_val.shape, y_val.shape)\n","\n","        # Define the model, re-initializing for each fold\n","        classif_model = build_mlp_model([X_tr.shape[1]])\n","        classif_model.compile(optimizer = optimizers.Adam(lr = learning_rate), \n","                              loss = loss)\n","        file_path = path + 'wts/pure_bert' + suffix + \"_{}{}{}.hdf5\".format(run,fold_n,fold_n_inner)\n","        callbacks = [kc.ModelCheckpoint(file_path, monitor = \"val_loss\", verbose = 0, save_best_only = True, mode = \"min\"),\n","                     kc.EarlyStopping(monitor='val_loss', patience=patience, restore_best_weights = True)]\n","\n","        # train the model\n","        classif_model.fit(x = X_tr, \n","                          y = y_tr, \n","                          epochs = epochs, \n","                          batch_size = batch_size, \n","                          callbacks = callbacks, \n","                          validation_data = (X_val, y_val), \n","                          verbose = 0)\n","\n","        for TTA_suffix in TTA_suffixes:\n","          pred = classif_model.predict(x = d_XY[TTA_suffix]['X_dev'], verbose = 0)\n","          pred_oof = classif_model.predict(x = X_oof_val_d[TTA_suffix], verbose=0)      \n","          sub_all_d[TTA_suffix].loc[sub_all_d[TTA_suffix].index[oof_val_index],['A','B','NEITHER']] += pred_oof / n_fold\n","          pred_all_d[TTA_suffix] += pred / n_fold / n_fold    \n","\n","    for TTA_suffix in TTA_suffixes:    \n","      # for Test\n","      sub_df = pd.read_csv(path+'input/gap-validation.tsv',sep='\\t').iloc[sanity_idx][['ID']]\n","      sub_df['A'] = 1/3; sub_df['B'] = 1/3; sub_df['NEITHER'] = 1/3 \n","\n","      sub_df.loc[:,['A','B','NEITHER']] = pred_all_d[TTA_suffix]      \n","      sub_df.to_csv(path+'sub/test_'+suffix+'_'+TTA_suffix+'_run{:d}_{:.5f}.csv'.format(run,log_loss(d_XY['orig']['Y_dev'], pred_all_d[TTA_suffix])), index=False)        \n","      print(f'run{run} {TTA_suffix} ' + \"{:d}folds {:.5f}\".format(n_fold, log_loss(d_XY['orig']['Y_dev'], pred_all_d[TTA_suffix]))) # Calculate the log loss \n","\n","      sub_all_d[TTA_suffix].to_csv(path+'sub/oof_'+suffix+'_'+TTA_suffix+'_run{:d}_{:.5f}.csv'.format(run,log_loss(d_XY['orig']['Y_train'], sub_all_d[TTA_suffix].loc[:,['A','B','NEITHER']].values)), index=False)\n","      print(f'run{run} {TTA_suffix} ' + \"{:d}folds OOF ================= {:.5f}\".format(n_fold, log_loss(d_XY['orig']['Y_train'], sub_all_d[TTA_suffix].loc[:,['A','B','NEITHER']]))) # Calculate the log loss    "],"execution_count":0,"outputs":[]},{"metadata":{"id":"Fu_9Fdl7AoLl","colab_type":"code","colab":{}},"cell_type":"code","source":["%%time\n","\n","#####  all_train\n","# pred_oof, sub_all_d: OOF\n","# pred_all_d, sub_df : sanity\n","\n","#####  sub_B\n","# pred_all_d, sub_df : dev\n","\n","if not all_train:\n","\n","  pd.DataFrame(columns=['a','b']).to_csv(path+'sub/tmp.csv') # testing drive connection\n","\n","  for run in range(0,5):\n","    import gc; gc.collect()\n","\n","    pred_all_d = {} # to save 25 fold avg (for Test), 5 outer OOF, 5 inner early stop\n","    for TTA_suffix in TTA_suffixes: pred_all_d[TTA_suffix] = np.zeros((num_test,3))        \n","\n","    # outer 5 fold: OOF fold. 4/5 train, 1/5 OOF pred  \n","    kfold = KFold(n_splits=n_fold, shuffle=True, random_state=3)\n","    for fold_n, (train_index, valid_index) in enumerate(kfold.split(d_XY['orig']['X_train'])):     \n","\n","      X_tr  = d_XY['orig']['X_train'][train_index,:]\n","      X_tr_orig = X_tr.copy()\n","      X_val = d_XY['orig']['X_train'][valid_index,:]\n","      y_tr  = d_XY['orig']['Y_train'][train_index,:]\n","      y_val = d_XY['orig']['Y_train'][valid_index,:]              \n","\n","      # train augmentation\n","      if len(Aug_suffixes)>1: \n","        patience = np.ceil(patience_orig / (1+len(Aug_suffixes)))\n","\n","        for k in Aug_suffixes:\n","          X_tr = np.concatenate((X_tr, d_XY[k]['X_train'][train_index,:]),axis=0)\n","          y_tr = np.concatenate((y_tr, y_tr),axis=0)\n","\n","        arr = np.arange(X_tr.shape[0])\n","        np.random.shuffle(arr)\n","        X_tr = X_tr[arr,:] \n","        y_tr = y_tr[arr,:]  \n","        print(X_tr.shape, y_tr.shape, X_val.shape, y_val.shape)\n","\n","      # Define the model, re-initializing for each fold\n","      classif_model = build_mlp_model([X_tr.shape[1]])\n","      classif_model.compile(optimizer = optimizers.Adam(lr = learning_rate), \n","                            loss = loss)\n","      file_path = path + 'wts/pure_bert' + suffix + \"_{}{}.hdf5\".format(run,fold_n)\n","      callbacks = [kc.ModelCheckpoint(file_path, monitor = \"val_loss\", verbose = 0, save_best_only = True, mode = \"min\"),\n","                   kc.EarlyStopping(monitor='val_loss', patience=patience, restore_best_weights = True)]\n","\n","      # train the model\n","      classif_model.fit(x = X_tr, \n","                        y = y_tr, \n","                        epochs = epochs, \n","                        batch_size = batch_size, \n","                        callbacks = callbacks, \n","                        validation_data = (X_val, y_val), \n","                        verbose = 0)\n","\n","      for TTA_suffix in TTA_suffixes:\n","        pred = classif_model.predict(x = d_XY[TTA_suffix]['X_dev'], verbose = 0)\n","        pred_all_d[TTA_suffix] += pred / n_fold    \n","\n","    for TTA_suffix in TTA_suffixes:    \n","      sub_df = pd.read_csv(path+'input/sample_submission_stage_1.csv')\n","\n","      sub_df.loc[:,['A','B','NEITHER']] = pred_all_d[TTA_suffix]      \n","      sub_df.to_csv(path+'sub/test_'+suffix+'_'+TTA_suffix+'_run{:d}_{:.5f}.csv'.format(run,log_loss(d_XY['orig']['Y_dev'], pred_all_d[TTA_suffix])), index=False)        \n","      print(f'run{run} {TTA_suffix} ' + \"{:d}folds {:.5f}\".format(n_fold, log_loss(d_XY['orig']['Y_dev'], pred_all_d[TTA_suffix]))) # Calculate the log loss "],"execution_count":0,"outputs":[]},{"metadata":{"id":"OTptHaKvDJJ_","colab_type":"code","colab":{}},"cell_type":"code","source":[""],"execution_count":0,"outputs":[]}]}