biGRU_attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "# import gensim\n",
    "import numpy as np\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import os\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import random\n",
    "import shutil\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "import re\n",
    "import gensim\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "import sys\n",
    "from scipy import sparse\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer, text_to_word_sequence\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.utils.np_utils import to_categorical\n",
    "import time\n",
    "\n",
    "from attention import * \n",
    "from tensorflow.python.ops.rnn import bidirectional_dynamic_rnn as bi_rnn\n",
    "from tensorflow.contrib.rnn import GRUCell"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SENT_LENGTH = 100\n",
    "MAX_SENTS = 100 # this is the longest number of sentences in a document!!!\n",
    "MAX_NB_WORDS = 60000\n",
    "EMBEDDING_DIM = 300 #due to word2vec dimension!!! \n",
    "HIDDEN_SIZE = 50 #based on Yang et al CMU. (Hierachical Attention Networks for Document Classification)\n",
    "ATTENTION_SIZE = 100 #same as Yang et al. \n",
    "BATCH_SIZE = 50\n",
    "NUM_ITERS = 10\n",
    "DISPLAY_STEP = 10\n",
    "VALIDATION_STEP = 10\n",
    "TESTING_BATCH = 10\n",
    "USER_EMBEDDING_DIM = 200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running time of loading Word2Vec model:  101.53218746185303\n"
     ]
    }
   ],
   "source": [
    "tic = time.time()\n",
    "# Download GoogleNews-vectors-negative300.bin.gz at \n",
    "#https://github.com/mmihaltz/word2vec-GoogleNews-vectors/blob/master/GoogleNews-vectors-negative300.bin.gz\n",
    "word2vec = gensim.models.KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin.gz', binary=True)\n",
    "toc = time.time()\n",
    "print(\"Running time of loading Word2Vec model: \", (toc - tic))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch_generator(X, y, batch_size):\n",
    "    \"\"\"Primitive batch generator \n",
    "    \"\"\"\n",
    "    size = X.shape[0]\n",
    "    X_copy = X.copy()\n",
    "    y_copy = y.copy()\n",
    "    indices = np.arange(size)\n",
    "    np.random.shuffle(indices)\n",
    "    X_copy = X_copy[indices]\n",
    "    y_copy = y_copy[indices]\n",
    "    i=0\n",
    "\n",
    "    while True:\n",
    "        left, right = i*batch_size, (i+1)*batch_size\n",
    "        right = min(size, right)\n",
    "        yield X_copy[left:right], y_copy[left:right]\n",
    "        if right >= size:\n",
    "            i = 0\n",
    "            indices = np.arange(size)\n",
    "            X_copy = X_copy[indices]\n",
    "            y_copy = y_copy[indices]\n",
    "        else:\n",
    "            i+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fit_tokenizer_based_on_training(train_folder, test_folder, dict_selected_words,\n",
    "                                   ground_truth_file='snopes_ground_truth.csv'):\n",
    "    '''Return tensor data of URL based solely on training interactions only!!!'''\n",
    "    \n",
    "    \n",
    "    df = pd.read_csv(ground_truth_file)\n",
    "    full = zip(df['snopes_page'], df['claim_label'])\n",
    "    dict_url_ground_truth = {}\n",
    "    for url, label in full:\n",
    "        assert label == True or label == False\n",
    "        if label == True:\n",
    "            dict_url_ground_truth[url] = 1\n",
    "        elif label == False:\n",
    "            dict_url_ground_truth[url] = 0\n",
    "        \n",
    "    assert len(dict_url_ground_truth) == 562\n",
    "    \n",
    "    def read_text_files(infolder):\n",
    "        documents = [fn for fn in listdir(infolder) if fn.endswith('.txt')]\n",
    "        data = np.zeros((len(documents), MAX_SENTS, MAX_SENT_LENGTH), dtype='int32')\n",
    "        dict_docs = {}\n",
    "        for fn in documents:\n",
    "            p = join(infolder, fn)\n",
    "            fin1 = open(p, 'r')\n",
    "            url = fin1.readline().replace('\\n', '')\n",
    "            assert 'http' in url\n",
    "            sents = []\n",
    "            for line in fin1:\n",
    "                sents.append(line.replace('\\n', ''))\n",
    "            dict_docs[url] = sents\n",
    "                \n",
    "        Y = []\n",
    "        for idx, url in enumerate(dict_docs.keys()):\n",
    "            assert 'http' in url\n",
    "            label = dict_url_ground_truth[url]\n",
    "            assert label == 0 or label == 1\n",
    "            Y.append([label, 1-label])\n",
    "            sentences = dict_docs[url]\n",
    "            for j, sent in enumerate(sentences):\n",
    "                if j < MAX_SENTS:\n",
    "                    wordTokens = text_to_word_sequence(sent)\n",
    "                    k = 0\n",
    "                    for _, word in enumerate(wordTokens):\n",
    "                        if word not in dict_selected_words:\n",
    "                            continue\n",
    "                        index_of_word = dict_selected_words[word]\n",
    "                        assert index_of_word >= 1 and index_of_word <= 16000\n",
    "                        if k < MAX_SENT_LENGTH:\n",
    "                            data[idx, j, k] = index_of_word\n",
    "                            k+=1\n",
    "                    \n",
    "        return data, np.array(Y)\n",
    "        \n",
    "    X_train, y_train = read_text_files(infolder=train_folder)\n",
    "    X_test, y_test = read_text_files(infolder=test_folder)\n",
    "    print(y_train.shape, \"here\")\n",
    "    return X_train, y_train, X_test, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_url_content_text2num(selected_words_file='out_top_16K_words_file.txt'):\n",
    "    parent = \"train_test_data\"\n",
    "    dict_folds = {}\n",
    "    fin = open(selected_words_file, 'r')\n",
    "    cnt = 1\n",
    "    dict_selected_words = {}\n",
    "    for line in fin:\n",
    "        _, w, _ = line.split()\n",
    "        dict_selected_words[w] = cnt\n",
    "        cnt += 1\n",
    "    assert len(dict_selected_words) == 16000 and max(dict_selected_words.values()) == 16000\n",
    "    fin.close()\n",
    "    \n",
    "    for i in range(0, 5):\n",
    "        #########################################################\n",
    "        dict_words = {} #for stat\n",
    "        data_i = '%s/data_%s' % (parent, i)\n",
    "        train_folder = '%s/train' % data_i\n",
    "        test_folder = '%s/test' % data_i\n",
    "        X_train, y_train, X_test, y_test = fit_tokenizer_based_on_training(train_folder, test_folder, dict_selected_words)\n",
    "        dict_folds[i] = (X_train, y_train, X_test, y_test)\n",
    "        \n",
    "    return dict_folds, dict_selected_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(448, 2) here\n",
      "(448, 2) here\n",
      "(448, 2) here\n",
      "(448, 2) here\n",
      "(448, 2) here\n"
     ]
    }
   ],
   "source": [
    "dict_folds, dict_selected_words = load_url_content_text2num()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_vocabs = 16000 \n",
    "'''Word2vec embedding matrix'''\n",
    "embedding_matrix = np.random.random((no_vocabs, EMBEDDING_DIM))\n",
    "#i starts at 1 not 0 like normal stuff!!!!\n",
    "for word, i in dict_selected_words.items(): \n",
    "    if word in word2vec.wv.vocab:\n",
    "        embedding_vector = word2vec[word]\n",
    "        embedding_matrix[i-1] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy at [iter 0][batch 0 of 8][cntStep: 0] : 0.5\n",
      "Validation accuracy is:  0.605263157895  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 1][batch 2 of 8][cntStep: 10] : 0.4\n",
      "Validation accuracy is:  0.508771929825  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 2][batch 4 of 8][cntStep: 20] : 0.56\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 3][batch 6 of 8][cntStep: 30] : 0.48\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 5][batch 0 of 8][cntStep: 40] : 0.5\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 6][batch 2 of 8][cntStep: 50] : 0.46\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 7][batch 4 of 8][cntStep: 60] : 0.46\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 8][batch 6 of 8][cntStep: 70] : 0.48\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 10][batch 0 of 8][cntStep: 80] : 0.541667\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 11][batch 2 of 8][cntStep: 90] : 0.58\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 12][batch 4 of 8][cntStep: 100] : 0.44\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 13][batch 6 of 8][cntStep: 110] : 0.56\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 15][batch 0 of 8][cntStep: 120] : 0.48\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 16][batch 2 of 8][cntStep: 130] : 0.5\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 17][batch 4 of 8][cntStep: 140] : 0.46\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 18][batch 6 of 8][cntStep: 150] : 0.46\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 20][batch 0 of 8][cntStep: 160] : 0.48\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 21][batch 2 of 8][cntStep: 170] : 0.541667\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 22][batch 4 of 8][cntStep: 180] : 0.58\n",
      "Validation accuracy is:  0.5  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 23][batch 6 of 8][cntStep: 190] : 0.5\n",
      "Validation accuracy is:  0.526315789474  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 25][batch 0 of 8][cntStep: 200] : 0.6\n",
      "Validation accuracy is:  0.517543859649  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 26][batch 2 of 8][cntStep: 210] : 0.88\n",
      "Validation accuracy is:  0.552631578947  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 27][batch 4 of 8][cntStep: 220] : 0.94\n",
      "Validation accuracy is:  0.561403508772  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 28][batch 6 of 8][cntStep: 230] : 0.94\n",
      "Validation accuracy is:  0.570175438596  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 30][batch 0 of 8][cntStep: 240] : 0.92\n",
      "Validation accuracy is:  0.657894736842  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 31][batch 2 of 8][cntStep: 250] : 1.0\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 32][batch 4 of 8][cntStep: 260] : 0.979167\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 33][batch 6 of 8][cntStep: 270] : 1.0\n",
      "Validation accuracy is:  0.649122807018  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 35][batch 0 of 8][cntStep: 280] : 0.98\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 36][batch 2 of 8][cntStep: 290] : 1.0\n",
      "Validation accuracy is:  0.666666666667  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 37][batch 4 of 8][cntStep: 300] : 1.0\n",
      "Validation accuracy is:  0.684210526316  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 38][batch 6 of 8][cntStep: 310] : 1.0\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 40][batch 0 of 8][cntStep: 320] : 1.0\n",
      "Validation accuracy is:  0.710526315789  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 41][batch 2 of 8][cntStep: 330] : 1.0\n",
      "Validation accuracy is:  0.666666666667  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 42][batch 4 of 8][cntStep: 340] : 1.0\n",
      "Validation accuracy is:  0.701754385965  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 43][batch 6 of 8][cntStep: 350] : 1.0\n",
      "Validation accuracy is:  0.701754385965  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 45][batch 0 of 8][cntStep: 360] : 1.0\n",
      "Validation accuracy is:  0.657894736842  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 46][batch 2 of 8][cntStep: 370] : 1.0\n",
      "Validation accuracy is:  0.684210526316  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 47][batch 4 of 8][cntStep: 380] : 1.0\n",
      "Validation accuracy is:  0.657894736842  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 48][batch 6 of 8][cntStep: 390] : 1.0\n",
      "Validation accuracy is:  0.701754385965  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 50][batch 0 of 8][cntStep: 400] : 1.0\n",
      "Validation accuracy is:  0.684210526316  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 51][batch 2 of 8][cntStep: 410] : 1.0\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 52][batch 4 of 8][cntStep: 420] : 1.0\n",
      "Validation accuracy is:  0.657894736842  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 53][batch 6 of 8][cntStep: 430] : 1.0\n",
      "Validation accuracy is:  0.675438596491  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 55][batch 0 of 8][cntStep: 440] : 1.0\n",
      "Validation accuracy is:  0.710526315789  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 56][batch 2 of 8][cntStep: 450] : 1.0\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 57][batch 4 of 8][cntStep: 460] : 1.0\n",
      "Validation accuracy is:  0.684210526316  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 58][batch 6 of 8][cntStep: 470] : 1.0\n",
      "Validation accuracy is:  0.657894736842  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 60][batch 0 of 8][cntStep: 480] : 1.0\n",
      "Validation accuracy is:  0.684210526316  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 61][batch 2 of 8][cntStep: 490] : 1.0\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 62][batch 4 of 8][cntStep: 500] : 1.0\n",
      "Validation accuracy is:  0.666666666667  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 63][batch 6 of 8][cntStep: 510] : 1.0\n",
      "Validation accuracy is:  0.719298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 65][batch 0 of 8][cntStep: 520] : 1.0\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 66][batch 2 of 8][cntStep: 530] : 1.0\n",
      "Validation accuracy is:  0.675438596491  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 67][batch 4 of 8][cntStep: 540] : 1.0\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 68][batch 6 of 8][cntStep: 550] : 1.0\n",
      "Validation accuracy is:  0.728070175439  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 70][batch 0 of 8][cntStep: 560] : 1.0\n",
      "Validation accuracy is:  0.701754385965  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 71][batch 2 of 8][cntStep: 570] : 1.0\n",
      "Validation accuracy is:  0.675438596491  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 72][batch 4 of 8][cntStep: 580] : 1.0\n",
      "Validation accuracy is:  0.728070175439  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 73][batch 6 of 8][cntStep: 590] : 1.0\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 75][batch 0 of 8][cntStep: 600] : 1.0\n",
      "Validation accuracy is:  0.710526315789  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 76][batch 2 of 8][cntStep: 610] : 1.0\n",
      "Validation accuracy is:  0.684210526316  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 77][batch 4 of 8][cntStep: 620] : 1.0\n",
      "Validation accuracy is:  0.701754385965  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 78][batch 6 of 8][cntStep: 630] : 1.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy is:  0.684210526316  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 80][batch 0 of 8][cntStep: 640] : 1.0\n",
      "Validation accuracy is:  0.701754385965  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 81][batch 2 of 8][cntStep: 650] : 1.0\n",
      "Validation accuracy is:  0.675438596491  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 82][batch 4 of 8][cntStep: 660] : 1.0\n",
      "Validation accuracy is:  0.736842105263  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 83][batch 6 of 8][cntStep: 670] : 1.0\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 85][batch 0 of 8][cntStep: 680] : 1.0\n",
      "Validation accuracy is:  0.675438596491  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 86][batch 2 of 8][cntStep: 690] : 1.0\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 87][batch 4 of 8][cntStep: 700] : 1.0\n",
      "Validation accuracy is:  0.684210526316  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 88][batch 6 of 8][cntStep: 710] : 1.0\n",
      "Validation accuracy is:  0.710526315789  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 90][batch 0 of 8][cntStep: 720] : 1.0\n",
      "Validation accuracy is:  0.701754385965  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 91][batch 2 of 8][cntStep: 730] : 1.0\n",
      "Validation accuracy is:  0.684210526316  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 92][batch 4 of 8][cntStep: 740] : 1.0\n",
      "Validation accuracy is:  0.69298245614  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 93][batch 6 of 8][cntStep: 750] : 1.0\n",
      "Validation accuracy is:  0.728070175439  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 95][batch 0 of 8][cntStep: 760] : 1.0\n",
      "Validation accuracy is:  0.675438596491  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 96][batch 2 of 8][cntStep: 770] : 1.0\n",
      "Validation accuracy is:  0.710526315789  on shape  (114, 100, 100)\n",
      "Training accuracy at [iter 97][batch 4 of 8][cntStep: 780] : 1.0\n",
      "Validation accuracy is:  0.710526315789  on shape  (114, 100, 100)\n"
     ]
    }
   ],
   "source": [
    "from attention import * \n",
    "from tensorflow.python.ops.rnn import bidirectional_dynamic_rnn as bi_rnn\n",
    "from tensorflow.contrib.rnn import GRUCell\n",
    "log_dir = \"HAM_log\"\n",
    "\n",
    "for key in dict_folds:\n",
    "    X_train, y_train, X_test, y_test = dict_folds[key]\n",
    "    train_batch_generator = batch_generator(X_train, y_train, BATCH_SIZE)\n",
    "    test_batch_generator = batch_generator(X_test, y_test, TESTING_BATCH)\n",
    "    \n",
    "    #clear log_dir\n",
    "    if os.path.exists(log_dir):\n",
    "        shutil.rmtree(log_dir)\n",
    "\n",
    "    \n",
    "    def doValidation(X_data, curr_sess, accuracy_test, batch_size, prefix, placeholder_input):\n",
    "        num_batches = X_data.shape[0] / float(batch_size)\n",
    "        num_batches = int(num_batches)\n",
    "        sum_acc = 0\n",
    "        cnt_rows = 0\n",
    "        batch_sent_ph, y_ph, sentence_length_ph, doc_actual_length_ph = placeholder_input\n",
    "        for i in range(num_batches+1):\n",
    "\n",
    "            x_batch, y_batch = next(test_batch_generator)\n",
    "            temp = np.sum(x_batch, axis=2)\n",
    "            #print(\"Test data shape: \", x_batch.shape)\n",
    "            doc_actual_lengths_test = np.count_nonzero(temp, axis=1) \n",
    "            x_batch_reshaped = x_batch.reshape(-1, MAX_SENT_LENGTH)\n",
    "            sentence_actual_lengths_test = np.count_nonzero(x_batch_reshaped, axis=1)\n",
    "            acc_result = curr_sess.run(accuracy_test, feed_dict={batch_sent_ph: x_batch_reshaped, \n",
    "                                        y_ph: y_batch, sentence_length_ph: sentence_actual_lengths_test,\n",
    "                                        doc_actual_length_ph: doc_actual_lengths_test})\n",
    "            sum_acc += acc_result\n",
    "            cnt_rows += x_batch.shape[0]\n",
    "        assert cnt_rows == X_data.shape[0], 'Mismatched rows_count: %s vs. %s' % (cnt_rows, X_data.shape[0])\n",
    "        rr = sum_acc/float(X_data.shape[0])\n",
    "        return rr\n",
    "\n",
    "    def hamBiRNN():\n",
    "        tf.reset_default_graph() \n",
    "\n",
    "        '''Variable with shape (no_vocabs, EMBEDDING_DIM) to get vectors in a sentence'''\n",
    "        embedding_matrix_variable = tf.Variable(embedding_matrix, trainable=True, dtype=tf.float32)\n",
    "        #print(embedding_matrix_variable.shape)\n",
    "        #print(X_train.shape)\n",
    "\n",
    "        '''We will take a bunch of sentences, where each sentence has length MAX_SENT_LENGTH\n",
    "        Ex: Two sentences: [[1,2,5,6,0], [3,5,3,6,0]] where numbers indicate a word. We will look up the \n",
    "        word vector for each word based on the number. \n",
    "        '''\n",
    "        #batch_sent_ph = tf.placeholder(tf.int32, [None, MAX_SENT_LENGTH]) \n",
    "        '''\n",
    "        Hope it work. After looking up, the shape should be (batch_size, MAX_SENTS, MAX_SENT_LENGTH, EMBEDDING_DIM)\n",
    "        However, since we need to use bi_rnn and we learn representation of sentences first. Therefore, we should use shape \n",
    "        (?, MAX_SENT_LENGTH, EMBEDDING_DIM) where \"?\" should be a multiple of MAX_SENTS\n",
    "        '''\n",
    "        batch_sent_ph = tf.placeholder(tf.int32, [None, MAX_SENT_LENGTH], name=\"batch_sent_ph\")\n",
    "        batch_sent_embedded = tf.nn.embedding_lookup(embedding_matrix_variable, batch_sent_ph)\n",
    "        y_ph = tf.placeholder(tf.float32, [None, 2], name=\"labels\")\n",
    "        sentence_length_ph = tf.placeholder(tf.int32, [None], name=\"sentence_length_ph\")\n",
    "        doc_actual_length_ph = tf.placeholder(tf.int32, [None], name=\"doc_actual_length_ph\")\n",
    "        #print(batch_sent_embedded)\n",
    "\n",
    "        '''We do not specify sequence_length. Therefore, the number of GRU cell in forward (same as backward) is MAX_SENT_LENGTH'''\n",
    "        with tf.variable_scope(\"first_bi_rnn\"):\n",
    "            rnn_outputs, _ = bi_rnn(GRUCell(HIDDEN_SIZE), GRUCell(HIDDEN_SIZE), \n",
    "                                    inputs=batch_sent_embedded, \n",
    "                                    sequence_length=sentence_length_ph, \n",
    "                                    dtype=tf.float32)\n",
    "        with tf.name_scope(\"attention_first_bi_rnn\"):\n",
    "            attention_output, alphas = attention(rnn_outputs, ATTENTION_SIZE, return_alphas=True)\n",
    "        with tf.name_scope(\"dropout_afterfirst_bi_rnn\"):\n",
    "            dropout_first_bi_rnn = tf.nn.dropout(attention_output, keep_prob=0.8)\n",
    "        with tf.name_scope(\"sent_bedding_after_first_birnn\"):\n",
    "            sent_bedding_after_first_birnn = tf.reshape(dropout_first_bi_rnn, shape=[-1, MAX_SENTS, 2*HIDDEN_SIZE])\n",
    "        ###########second bi-rnn-layer ############################\n",
    "        with tf.variable_scope(\"second_bi_rnn\"):\n",
    "            bi_rnn_sent_outputs, _ = bi_rnn(GRUCell(2*HIDDEN_SIZE), GRUCell(2*HIDDEN_SIZE), \n",
    "                                            inputs=sent_bedding_after_first_birnn, \n",
    "                                            sequence_length=doc_actual_length_ph,\n",
    "                                            dtype=tf.float32)\n",
    "        with tf.name_scope(\"attention_second_bi_rnn\"):\n",
    "            attention_output2, alphas = attention(bi_rnn_sent_outputs, ATTENTION_SIZE, return_alphas=True)\n",
    "        with tf.name_scope(\"dropout_after_second_bi_rnn\"):\n",
    "            dropout_second_bi_rnn = tf.nn.dropout(attention_output2, keep_prob=0.8)\n",
    "\n",
    "        with tf.name_scope(\"FC_layer\"):\n",
    "            W = tf.Variable(tf.random_normal([HIDDEN_SIZE * 4, 2], stddev=0.1))\n",
    "            b = tf.Variable(tf.constant(0.1, shape=[1]))\n",
    "            y_hat = tf.matmul(dropout_second_bi_rnn, W) + b\n",
    "        #y_hat = tf.squeeze(y_hat)\n",
    "\n",
    "        with tf.name_scope(\"softmax\"):\n",
    "            out_softmax = tf.nn.softmax(logits=y_hat)\n",
    "\n",
    "        with tf.name_scope(\"loss_cross_entropy\"):\n",
    "            loss = -tf.reduce_mean(tf.reduce_sum(tf.cast(y_ph, tf.float32) * tf.log(out_softmax), axis=1))\n",
    "        #loss = tf.reduce_sum(out_softmax)\n",
    "        A = tf.summary.scalar(\"Loss\", loss)\n",
    "\n",
    "        with tf.variable_scope(\"Traininig\"):\n",
    "            train_step = tf.train.AdamOptimizer(1e-4).minimize(loss=loss)\n",
    "\n",
    "        with tf.variable_scope(\"evaluation\"):\n",
    "            correct_prediction = tf.equal(tf.argmax(out_softmax, 1), tf.argmax(y_ph, 1))\n",
    "            accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "            accuracy_test = tf.reduce_sum(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "        valVar = tf.Variable(0.0, \"valVar\")\n",
    "        valVal_ph = tf.placeholder(tf.float32, [], name=\"independent\")\n",
    "        update_valVar = valVar.assign(valVal_ph)\n",
    "        mySummary = tf.summary.scalar(\"Validation\", update_valVar)\n",
    "\n",
    "        B = tf.summary.scalar(\"Accuracy\", accuracy)\n",
    "        summary_op = tf.summary.merge([A, B])\n",
    "        #summary_op = tf.summary.merge_all()\n",
    "\n",
    "        #### Testing model phat ##########################\n",
    "        #TODO\n",
    "        pre_val_acc = -1\n",
    "        time_val_acc_reduced = 0\n",
    "\n",
    "        NUM_ITERS = 100\n",
    "        placeholder_input = (batch_sent_ph, y_ph, sentence_length_ph, doc_actual_length_ph)\n",
    "        STOP_TRAINING = False\n",
    "        saver = tf.train.Saver()\n",
    "        with tf.Session() as sess:\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            saver.save(sess, 'saved_models/my_test_model')\n",
    "\n",
    "            writer = tf.summary.FileWriter(log_dir, graph=sess.graph)\n",
    "            cnt_step = 0\n",
    "            for i in range(NUM_ITERS):\n",
    "                num_batches = X_train.shape[0] / float(BATCH_SIZE)\n",
    "                num_batches = int(num_batches)\n",
    "                #num_batches = 11\n",
    "                if STOP_TRAINING:\n",
    "                    break\n",
    "                for b in range(num_batches):\n",
    "                    if STOP_TRAINING:\n",
    "                        break\n",
    "                    x_batch, y_batch = next(train_batch_generator)\n",
    "                    temp = np.sum(x_batch, axis=2)\n",
    "                    doc_actual_lengths_better = np.count_nonzero(temp, axis=1) \n",
    "\n",
    "                    #when reshaping data for feeddict, you should use np.reshape\n",
    "                    x_batch = x_batch.reshape(-1, MAX_SENT_LENGTH)\n",
    "                    '''Actual length of sentences in this batch_size * so_luong_sentence_moi_doc'''\n",
    "                    sentence_actual_lengths_better = np.count_nonzero(x_batch, axis=1)\n",
    "                    if cnt_step % DISPLAY_STEP == 0:\n",
    "                        #print(\"At iter %s and batch %s of %s - cntStep: %s\" % (i, b, num_batches, cnt_step))\n",
    "                        train_acc = accuracy.eval(feed_dict={batch_sent_ph: x_batch, y_ph: y_batch,\n",
    "                                                           sentence_length_ph: sentence_actual_lengths_better,\n",
    "                                                           doc_actual_length_ph: doc_actual_lengths_better\n",
    "                                                           })\n",
    "                        print(\"Training accuracy at [iter %s][batch %s of %s][cntStep: %s] : %s\" % \n",
    "                              (i, b, num_batches, cnt_step, train_acc))\n",
    "                    ####################################################\n",
    "                    ### VALIDATION STEP TO AVOID OVERFITTING!!!!\n",
    "                    ########################################################\n",
    "                    if cnt_step % VALIDATION_STEP == 0:\n",
    "                        #do validation\n",
    "                        curr_val_acc = doValidation(X_data=X_test, curr_sess=sess,\n",
    "                                     accuracy_test=accuracy_test, batch_size=TESTING_BATCH, prefix=\"Validation\",\n",
    "                                    placeholder_input=placeholder_input)\n",
    "                        print('Validation accuracy is: ', curr_val_acc, ' on shape ', X_test.shape)\n",
    "\n",
    "                        _, ss = sess.run([update_valVar, mySummary], feed_dict={valVal_ph: curr_val_acc})\n",
    "                        writer.add_summary(ss, cnt_step)\n",
    "\n",
    "                        if curr_val_acc >= pre_val_acc:\n",
    "                            curr_val_acc = pre_val_acc\n",
    "                            time_val_acc_reduced = 0\n",
    "                        else:\n",
    "                            time_val_acc_reduced +=1\n",
    "                            if time_val_acc_reduced >= 10:\n",
    "                                #10 times accuracy reduced over time, we stop training!!!!\n",
    "                                print(\"Validation reduced!!! We should stop training here!!!!\")\n",
    "                                STOP_TRAINING = True\n",
    "\n",
    "                    #assert x_batch.shape == (BATCH_SIZE * MAX_SENTS, MAX_SENT_LENGTH), x_batch.shape\n",
    "                    summary, _ = sess.run([summary_op, train_step], \n",
    "                                                 feed_dict={batch_sent_ph: x_batch, y_ph: y_batch,\n",
    "                                                           sentence_length_ph: sentence_actual_lengths_better,\n",
    "                                                           doc_actual_length_ph: doc_actual_lengths_better\n",
    "                                                           })\n",
    "                    writer.add_summary(summary, cnt_step)\n",
    "                    cnt_step += 1\n",
    "\n",
    "            writer.close()\n",
    "            #testing data\n",
    "            testing_acc = doValidation(X_data=X_test, curr_sess=sess, \n",
    "                         accuracy_test=accuracy_test, batch_size=TESTING_BATCH, prefix=\"Testing\",\n",
    "                        placeholder_input=placeholder_input)\n",
    "            print('Testing accuracy is: ', testing_acc, ' on shape ', X_test.shape)\n",
    "    hamBiRNN()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}