evi_bert.ipynb

{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "colab": {
   "name": "Bert_GitHub.ipynb",
   "provenance": [],
   "authorship_tag": "ABX9TyOEzBYcz+DxUEJVvSi61yJs",
   "include_colab_link": true
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "accelerator": "GPU"
 },
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "view-in-github",
    "colab_type": "text"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/TillJohanndeiter/evidence-quality-bert/blob/master/evi_bert.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "R5nw7cjdAjLh"
   },
   "source": [
    "If the code doesn't work caused by updates of tensorflow in the future, please uncomment the first four lines.\n"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "Wut2HPT1U89t"
   },
   "source": [
    "\"\"\"\n",
    "!pip uninstall tensorflow\n",
    "!pip uninstall tensorflow_hub\n",
    "!pip install tensorflow==2.3.0\n",
    "!pip install tensorflow_hub==0.9.0\n",
    "\"\"\"\n",
    "!pip install bert-for-tf2"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "sU6cRm4Tu4s_"
   },
   "source": [
    "import pathlib\n",
    "import numpy\n",
    "import csv\n",
    "import tensorflow as tf\n",
    "import tensorflow.keras as keras\n",
    "import tensorflow_hub as hub\n",
    "import pandas\n",
    "from tensorflow.keras import Model\n",
    "from tensorflow.keras.layers import Input, Dense, GlobalAveragePooling1D\n",
    "from tensorflow.keras.metrics import BinaryAccuracy, Precision, Recall,\\\n",
    "    TruePositives, TrueNegatives, FalsePositives, FalseNegatives\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import TensorBoard, EarlyStopping\n",
    "from bert import bert_tokenization\n",
    "from pathlib import Path\n",
    "from numpy import int32"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yR_pYwrw-4aD"
   },
   "source": [
    "Download dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "YGW1I1Alu9f-"
   },
   "source": [
    "!wget --output-document=data.zip https://www.research.ibm.com/haifa/dept/vst/files/IBM_Debater_\\(R\\)_EviConv-ACL-2019.v1.zip\n",
    "!unzip data.zip\n",
    "!mv IBM_Debater_\\(R\\)_EviConv-ACL-2019.v1/*.csv ."
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "5hUUsHNzvEHc"
   },
   "source": [
    "MAX_SEQ_LENGTH = 100\n",
    "BERT_MODEL_HUB = \"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2\"\n",
    "BERT_LAYER = hub.KerasLayer(BERT_MODEL_HUB, trainable=True)\n",
    "\n",
    "vocab_file = BERT_LAYER.resolved_object.vocab_file.asset_path.numpy()\n",
    "do_lower_case = BERT_LAYER.resolved_object.do_lower_case.numpy()\n",
    "tokenizer = bert_tokenization.FullTokenizer(vocab_file=vocab_file, do_lower_case=do_lower_case)"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "X2lYWsihBHfu"
   },
   "source": [
    "**Preprocessing into bert readable format**\n",
    "\n",
    "The code, especially tokenization and formatting are inspired form official [bert repository](https://github.com/google-research/bert/blob/master/run_classifier.py).\n",
    "Method convert_examples_to_features and associated help methods were customized to the domain of dataset and unnecessary code parts are removed."
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "klNoItK-4KAc"
   },
   "source": [
    "class EviPair():\n",
    "    \"\"\"A single pair of two evidence which are ranked by the label.\"\"\"\n",
    "\n",
    "    def __init__(self,\n",
    "                 first_evi: str,\n",
    "                 second_evi: str,\n",
    "                 first_stance: str,\n",
    "                 second_stance: str,\n",
    "                 topic: str,\n",
    "                 acceptance_rate: str,\n",
    "                 label: int, ):\n",
    "        \"\"\"\n",
    "        evis:\n",
    "          first_evi: string. The untokenize text of the first evidence from the dataset.\n",
    "          second_evi: string. The untokenize text of the second evidence from the dataset.\n",
    "          label: string. The label of the data sample. Label equals one means first one is\n",
    "          the better evidence. Label equals second means second evidence is better.\n",
    "        \"\"\"\n",
    "        self.first_evi = first_evi\n",
    "        self.second_evi = second_evi\n",
    "        self.first_stance = first_stance\n",
    "        self.second_stance = second_stance\n",
    "        self.topic = topic\n",
    "        self.acceptance_rate = acceptance_rate\n",
    "        self.label = label\n",
    "\n",
    "\n",
    "class ProcessedEviPair():\n",
    "    \"\"\"Representation of tokenized and process evidence pair. Contains arrays of corresponding\n",
    "  token_ids, which makes an sentence understandable by bert. Input_mask shows which part of\n",
    "  token_ids are padded. Segment ids signalize which part of token_ids belong to first and second\n",
    "  evidence.\"\"\"\n",
    "\n",
    "    def __init__(self,\n",
    "                 token_ids,\n",
    "                 input_mask,\n",
    "                 segment_ids,\n",
    "                 label_id, ):\n",
    "        self.token_ids = token_ids\n",
    "        self.input_mask = input_mask\n",
    "        self.segment_ids = segment_ids\n",
    "        self.label = label_id\n",
    "\n",
    "\n",
    "def _truncate_evi_pair(tokens_a: [str], tokens_b: [str]):\n",
    "    \"\"\"Truncates a sequence pair in place to the maximum length.\"\"\"\n",
    "    # This is a simple heuristic which will always truncate the longer sequence\n",
    "    # one token at a time. This makes more sense than truncating an equal percent\n",
    "    # of tokens from each, since if one sequence is very short then each token\n",
    "    # that's truncated likely contains more information than a longer sequence.\n",
    "\n",
    "    while True:\n",
    "        total_length = len(tokens_a) + len(tokens_b)\n",
    "        if total_length <= MAX_SEQ_LENGTH - 3:\n",
    "            break\n",
    "        if len(tokens_a) > len(tokens_b):\n",
    "            tokens_a.pop()\n",
    "        else:\n",
    "            tokens_b.pop()\n",
    "\n",
    "\n",
    "def _process_evi_pair(evi_pair: EviPair) -> ProcessedEviPair:\n",
    "    \"\"\"\n",
    "    The convention in BERT is:\n",
    "    For sequence pairs:\n",
    "    tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]\n",
    "    input_ids: Corresponding numerical ids for each token.\n",
    "    input_mask : mask has 1 for real tokens and 0 for padding tokens\n",
    "    segment_ids: 0     0  0    0    0     0       0 0     1  1  1  1   1 1\n",
    "    Segment_ids are used to indicate whether this is the first\n",
    "    sequence or the second sequence.\n",
    "    Input_ids are used as sentence representation.\n",
    "    :param evi_pair: currently processed evidence pair\n",
    "    :return: ProcessedEviPair of evi_pair\n",
    "    \"\"\"\n",
    "\n",
    "    first_evi_tokens = tokenizer.tokenize(evi_pair.first_evi)\n",
    "    second_evi_tokens = tokenizer.tokenize(evi_pair.second_evi)\n",
    "\n",
    "    _truncate_evi_pair(first_evi_tokens, second_evi_tokens)\n",
    "\n",
    "    segment_ids, tokens = create_tokens(first_evi_tokens, second_evi_tokens)\n",
    "\n",
    "    input_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "\n",
    "    # 1 is appended len of input_ids to mark non padded part of input\n",
    "    input_mask = [1] * len(input_ids)\n",
    "\n",
    "    _pad_up_to_max_seq_len(input_ids, input_mask, segment_ids)\n",
    "\n",
    "    assert len(input_ids) == MAX_SEQ_LENGTH\n",
    "    assert len(input_mask) == MAX_SEQ_LENGTH\n",
    "    assert len(segment_ids) == MAX_SEQ_LENGTH\n",
    "\n",
    "    # Binary label is corrected form [1, 2] to [0,1]\n",
    "    return ProcessedEviPair(\n",
    "        token_ids=input_ids,\n",
    "        input_mask=input_mask,\n",
    "        segment_ids=segment_ids,\n",
    "        label_id=evi_pair.label - 1)\n",
    "\n",
    "\n",
    "def create_tokens(first_evi_tokens, sec_evi_tokens) -> ([], []):\n",
    "    \"\"\"\n",
    "    Create segments_ids (0 for first evidence, 1 for second evidence)\n",
    "    and add [SEP] and [CLS] and merge first_evi_tokens and sec_evi_tokens\n",
    "    :param first_evi_tokens: tokens of first evidence\n",
    "    :param sec_evi_tokens: second of first evidence\n",
    "    :return: segment_ids and tokens for both evidences\n",
    "    \"\"\"\n",
    "    tokens = []\n",
    "    segment_ids = []\n",
    "    tokens.append(\"[CLS]\")\n",
    "    segment_ids.append(0)\n",
    "    for token in first_evi_tokens:\n",
    "        tokens.append(token)\n",
    "        segment_ids.append(0)\n",
    "    tokens.append(\"[SEP]\")\n",
    "    segment_ids.append(0)\n",
    "    if sec_evi_tokens:\n",
    "        for token in sec_evi_tokens:\n",
    "            tokens.append(token)\n",
    "            segment_ids.append(1)\n",
    "        tokens.append(\"[SEP]\")\n",
    "        segment_ids.append(1)\n",
    "    return segment_ids, tokens\n",
    "\n",
    "\n",
    "def _pad_up_to_max_seq_len(input_ids, input_mask, segment_ids):\n",
    "    \"\"\"\n",
    "    Add zeros to input_ids\n",
    "    :param input_ids: input_ids for currently processed evidence\n",
    "    :param input_mask: input_mask for currently processed evidence\n",
    "    :param segment_ids: segment_ids for currently processed evidence\n",
    "    :return: None\n",
    "    \"\"\"\n",
    "    while len(input_ids) < MAX_SEQ_LENGTH:\n",
    "        input_ids.append(0)\n",
    "        input_mask.append(0)\n",
    "        segment_ids.append(0)\n",
    "\n",
    "\n",
    "def _process_evi_pairs(evi_pairs: []):\n",
    "    \"\"\"Convert a set of eviPairs to a list of ProcessedEviPair.\"\"\"\n",
    "    return [_process_evi_pair(example) for example in evi_pairs]\n",
    "\n",
    "\n",
    "def _convert_to_numpy_arrays(evidences: [ProcessedEviPair]) \\\n",
    "        -> ((numpy.ndarray, numpy.ndarray, numpy.ndarray), numpy.ndarray):\n",
    "    \"\"\"\n",
    "    Convert list of ProcessedEviPair's to 4 numpy arrays which are compatible with\n",
    "    keras and tensorflow\n",
    "    :param evidences: list of ProcessedEviPair's to convert\n",
    "    :return: tuples of one and three numpy arrays (token_ids, attention_mask, type_ids) labels\n",
    "    \"\"\"\n",
    "    return (numpy.stack([evi.token_ids for evi in evidences]),\n",
    "            numpy.stack([evi.input_mask for evi in evidences], ),\n",
    "            numpy.stack([evi.segment_ids for evi in evidences], )), \\\n",
    "           numpy.stack([evi.label for evi in evidences])\n",
    "\n",
    "\n",
    "def x_and_y_from_dataset(filepath: Path) -> \\\n",
    "        ((numpy.ndarray, numpy.ndarray, numpy.ndarray), numpy.ndarray):\n",
    "    \"\"\"\n",
    "    Main function which receives absolute or relative filepath of an csv file\n",
    "    which will be tokenized, type/attention masks and labels are created. Furthermore\n",
    "    a tuple of numpy arrays is created which is directly usable for keras.\n",
    "    :param filepath: absolute or relative path of csv file\n",
    "    :return: tuples of one and three numpy arrays (token_ids, attention_mask, type_ids) labels\n",
    "    \"\"\"\n",
    "\n",
    "    # Create instance of EviPair with correct label for\n",
    "    evi_pairs = load_evi_pairs(filepath)\n",
    "    processed_evi_pairs = _process_evi_pairs(evi_pairs)\n",
    "    x_numpy, y_numpy = _convert_to_numpy_arrays(processed_evi_pairs)\n",
    "\n",
    "    # Check that entry and y have same length\n",
    "    assert len(x_numpy[0]) == len(x_numpy[1]) == len(x_numpy[2]) == len(y_numpy)\n",
    "\n",
    "    # Check that all input vectors have same length as MAX_SEQ_LENGTH\n",
    "    assert x_numpy[0].shape[1] == x_numpy[1].shape[1] == x_numpy[2].shape[1] == MAX_SEQ_LENGTH\n",
    "\n",
    "    return x_numpy, y_numpy\n",
    "\n",
    "\n",
    "def x_and_y_from_evi_pair(evi_pair: EviPair) -> \\\n",
    "        ((numpy.ndarray, numpy.ndarray, numpy.ndarray), numpy.ndarray):\n",
    "    \"\"\"\n",
    "    Convert evi_pair to keras usable numpy arrays\n",
    "    :param evi_pair: Sample from dataset which will be converted\n",
    "    :return: numpy arrays which can be used with keras\n",
    "    \"\"\"\n",
    "    process_evi_pair = _process_evi_pair(evi_pair)\n",
    "    return _convert_to_numpy_arrays([process_evi_pair])\n",
    "\n",
    "\n",
    "def load_evi_pairs(filepath) -> [EviPair]:\n",
    "    \"\"\"\n",
    "    Load csv dataset from filepath and convert it to dataframe and\n",
    "    afterwards to list of EviPair's.\n",
    "    :param filepath: filepath of csv dataset to parse\n",
    "    :return: list of EviPair's\n",
    "    \"\"\"\n",
    "    dataframe = pandas.read_csv(filepath)\n",
    "    evi_pairs = dataframe.apply(lambda entry: EviPair(first_evi=entry[1],\n",
    "                                                      second_evi=entry[2],\n",
    "                                                      topic=entry[0],\n",
    "                                                      first_stance=entry[5],\n",
    "                                                      second_stance=entry[6],\n",
    "                                                      acceptance_rate=entry[4],\n",
    "                                                      label=entry[3]), axis=1)\n",
    "    return evi_pairs.tolist()\n"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-0wtZ4-QBS51"
   },
   "source": [
    "**Load dataset from specified folder**"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "D2oi6XYJvJRx"
   },
   "source": [
    "train_x, train_y = x_and_y_from_dataset(pathlib.Path('train.csv'))\n",
    "test_x, test_y = x_and_y_from_dataset(pathlib.Path('test.csv'))"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "oJa9TTx4TOAo"
   },
   "source": [
    "METRICS = [BinaryAccuracy(),\n",
    "           Precision(),\n",
    "           Recall(),\n",
    "           TruePositives(),\n",
    "           TrueNegatives(),\n",
    "           FalsePositives(),\n",
    "           FalseNegatives()]\n",
    "\n",
    "def evi_bert() -> Model:\n",
    "    \"\"\"\n",
    "    Creates model using pretrained bert layer form google. Has three inputs\n",
    "    (word_ids, masks and segment_ids). For explanation please look into\n",
    "    documentation form preprocessing. Print summary after compilation.\n",
    "    :return: created model\n",
    "    \"\"\"\n",
    "\n",
    "    input_word_ids = Input(shape=(MAX_SEQ_LENGTH,), dtype=int32,\n",
    "                                  name=\"input_word_ids\")\n",
    "    input_mask = Input(shape=(MAX_SEQ_LENGTH,), dtype=int32,\n",
    "                              name=\"input_mask\")\n",
    "    segment_ids = Input(shape=(MAX_SEQ_LENGTH,), dtype=int32,\n",
    "                               name=\"segment_ids\")\n",
    "    bert_inputs = [input_word_ids, input_mask, segment_ids]\n",
    "\n",
    "    _, sequence_output = BERT_LAYER(bert_inputs)\n",
    "    sequence_output = GlobalAveragePooling1D(name='average_over_time')(sequence_output)\n",
    "\n",
    "    output = Dense(1, activation='sigmoid')(sequence_output)\n",
    "\n",
    "    model = Model(inputs=bert_inputs, outputs=output, name='evi_bert')\n",
    "    model.compile(loss='binary_crossentropy', optimizer=Adam(learning_rate=1e-5),\n",
    "                  metrics=METRICS)\n",
    "\n",
    "    return model\n"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "ZNMoH2EmRLhP"
   },
   "source": [
    "evi_bert = evi_bert()\n",
    "evi_bert.fit(x=train_x, y=train_y,\n",
    "           batch_size=16, epochs=10, callbacks=[TensorBoard(), EarlyStopping()], validation_split=0.2, shuffle=True)\n",
    "\n",
    "scores = evi_bert.evaluate(x=test_x, y=test_y, verbose=1)\n",
    "print('Binary Accuracy: {}'.format(scores[1]))\n",
    "print('Precision: {}'.format(scores[2]))\n",
    "print('Recall: {}'.format(scores[3]))\n",
    "\n",
    "# Print values of confusion matrix\n",
    "\n",
    "print('True negatives / Labeled with first evidence '\n",
    "      'and predicted as first evidence  : {}'.format(scores[5]))\n",
    "print('True positives / Labeled with second evidence '\n",
    "      'and predicted as second evidence : {}'.format(scores[4]))\n",
    "print('False negatives / Labeled with first evidence '\n",
    "      'and predicted as second evidence : {}'.format(scores[7]))\n",
    "print('False positives / Labeled with second evidence '\n",
    "      'and predicted as first evidence : {}'.format(scores[6]))"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "n8_HSZq8_bw0"
   },
   "source": [
    "**Optional if you want to save your model at google drive**\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "zsUl4XNlnKOU"
   },
   "source": [
    "evi_bert.save('gdrive/My Drive/MODEL_SAVE_PATH')"
   ],
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "X4MzhPWB_vTH"
   },
   "source": [
    "**Optional if you want to use TensorBoard**"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "id": "xcPSSHCghbA0"
   },
   "source": [
    "%tensorboard --logdir logs"
   ],
   "execution_count": null,
   "outputs": []
  }
 ]
}