English_Twi_Neural_Machine_Translation_with_attention_3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "2zY7TrNFZTHH",
    "outputId": "24cfc0d7-47ca-4f73-a198-8227e5ed7054"
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "IPython.notebook.set_autosave_interval(5000)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autosaving every 5 seconds\n"
     ]
    }
   ],
   "source": [
    "#specify tensorflow version to use\n",
    "#%tensorflow_version 2.x\n",
    "#load tensorboard\n",
    "#%load_ext tensorboard\n",
    " #%tensorboard --logdir logs\n",
    "import tensorflow as tf\n",
    "%autosave 5\n",
    "#gpu_options = tf.GPUOptions(allow_growth=True)\n",
    "#session = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_options))\n",
    "#session = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction)\n",
    "#from tf.keras.layers.experimental.preprocessing import TextVectorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Yt4BurtWjtEV"
   },
   "outputs": [],
   "source": [
    "#drive.flush_and_unmount()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 54
    },
    "colab_type": "code",
    "id": "CPp9jzVtY0DK",
    "outputId": "422c9e26-3d45-4374-89e0-05d8ffea5dc1"
   },
   "outputs": [],
   "source": [
    "#from google.colab import drive\n",
    "#drive.mount('/content/drive')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bCiCUpTpSLC7"
   },
   "outputs": [],
   "source": [
    "#If you want to use the file and not have to remember to close it afterward, do this:\n",
    "\n",
    "\n",
    "with open('/home/nanaboateng/notebooks/Data/jw300.en-tw.tw', 'r') as f:\n",
    "     #Twi_data = f.readlines()\n",
    "      Twi_data= f.read().splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "id": "AZL_76fsaerK",
    "outputId": "bb904e87-0fe8-4318-8fb5-7dcfc0043d46"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['“ Oo , Yehowa , Boa Me Babea Kumaa Yi Ma Onni Nokware ! ”',\n",
       " 'WƆWOO me too abusua a wonim adwinne di mu wɔ Alsace , France , wɔ 1930 mu .',\n",
       " 'Ná Papa taa pa twere n’agua mu kenkan asase ho nsɛm anaa ewim nneɛma ho nhoma bi anwummere anwummere .',\n",
       " 'Ná me kraman da Papa nan so , na na ɔka nsɛntitiriw a epue wɔ n’akenkan no mu kyerɛ Maame bere a ɔnwene abusua no nneɛma no .',\n",
       " 'Ná m’ani gye anwummere a ɛtete saa no ho kɛse !']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Twi_data[0:5] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "id": "4HAkrCLGSK8y",
    "outputId": "4b1a352f-1d0d-48c5-9729-903b54e334e1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['“ Oh , Jehovah , Keep My Young Girl Faithful ! ”',\n",
       " 'I WAS born in 1930 in Alsace , France , into an artistic family .',\n",
       " 'During the evenings , Father , sitting in his lounge chair , would be reading some books about geography or astronomy .',\n",
       " 'My doggy would be sleeping by his feet , and Daddy would be sharing with Mum some highlights from his reading while she was knitting for her family .',\n",
       " 'How much I enjoyed those evenings !']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "english_data = [line.rstrip() for line in open('/home/nanaboateng/notebooks/Data/jw300.en-tw.en')]\n",
    "english_data[0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9CIOAxoOw35p"
   },
   "source": [
    "#### sequence to sequence\n",
    "encoder decoder model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "jxL44_zLw_rS",
    "outputId": "21d3471f-5a73-45df-80a9-d9e1cd26de3b"
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tfa' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-6ad9bd3cfccc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m#import tensorflow_addons as tfa\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtfa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__version__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;31m#print(dir(tfa.seq2seq))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'tfa' is not defined"
     ]
    }
   ],
   "source": [
    "#import tensorflow_addons as tfa\n",
    "print(tfa.__version__)\n",
    "#print(dir(tfa.seq2seq))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "tnxXKDjq3jEL",
    "outputId": "0275ea95-abb2-42d7-9ddb-c88e4fc1c3a8"
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "\n",
    "try:\n",
    "  # %tensorflow_version only exists in Colab.\n",
    "  %tensorflow_version 2.x\n",
    "except Exception:\n",
    "  pass\n",
    "import tensorflow as tf\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import unicodedata\n",
    "import re\n",
    "import numpy as np\n",
    "import os\n",
    "import io\n",
    "import time\n",
    "# encoding=utf8\n",
    "from importlib import reload\n",
    "import sys\n",
    "reload(sys)\n",
    "#sys.setdefaultencoding('utf8')\n",
    "%autosave 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rd0jw-eC3jEh"
   },
   "outputs": [],
   "source": [
    "def normalize_eng(s):\n",
    "    s = unicode_to_ascii(s)\n",
    "    s = re.sub(r'([!.?])', r' \\1', s)\n",
    "    s = re.sub(r'[^a-zA-Z.!?]+', r' ', s)\n",
    "    s = re.sub(r'\\s+', r' ', s)\n",
    "    return s\n",
    "\n",
    "def normalize_twi(s):\n",
    "    s = unicode_to_ascii(s)\n",
    "    s = re.sub(r'([!.?])', r' \\1', s)\n",
    "    s = re.sub(r'[^a-zA-Z.ƆɔɛƐ!?’]+', r' ', s)\n",
    "    s = re.sub(r'\\s+', r' ', s)\n",
    "    return s\n",
    "\n",
    "\n",
    "# Converts the unicode file to ascii\n",
    "def unicode_to_ascii(s):\n",
    "  return ''.join(c for c in unicodedata.normalize('NFD', s)\n",
    "      if unicodedata.category(c) != 'Mn')\n",
    "\n",
    "\n",
    "def preprocess_sentence_english(w):\n",
    "  w = unicode_to_ascii(w.lower().strip())\n",
    "\n",
    "  # creating a space between a word and the punctuation following it\n",
    "  # eg: \"he is a boy.\" => \"he is a boy .\"\n",
    "  # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
    "  w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "  w = re.sub(r'[\" \"]+', \" \", w)\n",
    "\n",
    "  # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\")\n",
    "  w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n",
    "  #w = re.sub(r'[^Ɔ-ɔɛƐ]+', r' ', w)\n",
    "  #strip() Parameters\n",
    "  #chars (optional) - a string specifying the set of characters to be removed.\n",
    "  #If the chars argument is not provided, all leading and trailing whitespaces are removed from the string.\n",
    "  w = w.rstrip().strip()\n",
    "\n",
    "  # adding a start and an end token to the sentence\n",
    "  # so that the model know when to start and stop predicting.\n",
    "  w = '<start> ' + w + ' <end>'\n",
    "  return w\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "  \n",
    "def preprocess_sentence_twi(w):\n",
    "  w = unicode_to_ascii(w.lower().strip())\n",
    "\n",
    "  # creating a space between a word and the punctuation following it\n",
    "  # eg: \"he is a boy.\" => \"he is a boy .\"\n",
    "  # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
    "  w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "  w = re.sub(r'[\" \"]+', \" \", w)\n",
    "\n",
    "  # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\")\n",
    "  w = re.sub(r\"[^a-zA-ZɛƐɔƆ?.!,¿]+\", \" \", w)\n",
    "  #w = re.sub(r'[^Ɔ-ɔɛƐ]+', r' ', w)\n",
    "  #strip() Parameters\n",
    "  #chars (optional) - a string specifying the set of characters to be removed.\n",
    "  #If the chars argument is not provided, all leading and trailing whitespaces are removed from the string.\n",
    "  w = w.rstrip().strip()\n",
    "\n",
    "  # adding a start and an end token to the sentence\n",
    "  # so that the model know when to start and stop predicting.\n",
    "  w = '<start> ' + w + ' <end>'\n",
    "  return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "id": "UkLhlUiChXtN",
    "outputId": "e928bea5-a736-46e9-9024-d4d1ad79a8c4"
   },
   "outputs": [],
   "source": [
    "type(english_data)\n",
    "\n",
    "english_d  = list(map(preprocess_sentence_english,english_data))\n",
    "english_d[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "id": "NJw9rV8AwQ3U",
    "outputId": "17def880-f244-4cc8-b5ce-cfb7baa8f163"
   },
   "outputs": [],
   "source": [
    "twi_d  = list(map(preprocess_sentence_twi,Twi_data))\n",
    "twi_d[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 102
    },
    "colab_type": "code",
    "id": "J-mhvNEJ8UYL",
    "outputId": "f18e23bb-ca06-464b-e696-ec04ac888b81"
   },
   "outputs": [],
   "source": [
    "\n",
    "all_data= (twi_d[0:30000]),(english_d[0:30000])\n",
    "all_data[0][0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BSwJEOpTsTNq"
   },
   "outputs": [],
   "source": [
    "def create_dataset_eng(path, num_examples):\n",
    "  lines = io.open(path, encoding='UTF-8').read().strip().split('\\n')\n",
    "\n",
    "  word_pairs = [[preprocess_sentence_english(w) for w in l.split('\\t')]  for l in lines[:num_examples]]\n",
    "\n",
    "  return zip(*word_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qPoajtCJtd7l"
   },
   "outputs": [],
   "source": [
    "#encoding = ‘utf-8-sig’ is added to overcome the issue when exporting ‘Non-English’ languages.\n",
    "def create_dataset_twi(path, num_examples):\n",
    "  lines = io.open(path, encoding='utf-8-sig').read().strip().split('\\n')\n",
    "\n",
    "  word_pairs = [[preprocess_sentence_twi(w) for w in l.split('\\t')]  for l in lines[:num_examples]]\n",
    "\n",
    "  return zip(*word_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "d6UOWPlCsoSp",
    "outputId": "a7bf8edd-4086-44ff-ad99-a607002fbca3"
   },
   "outputs": [],
   "source": [
    "num_examples=3000\n",
    "path_eng='/content/drive/My Drive/NaturalLanguageProcessing/GhanaNLP/jw300.en-tw.en' \n",
    "\n",
    "eng=create_dataset_eng(path_eng, num_examples)\n",
    "print(eng)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "8FkRW8pOtHu1",
    "outputId": "3efe2cbe-4a4e-4ac1-8fcf-2ab4b6a20fa5"
   },
   "outputs": [],
   "source": [
    "path_twi = '/content/drive/My Drive/NaturalLanguageProcessing/GhanaNLP/jw300.en-tw.tw'\n",
    "twi=create_dataset_twi(path_twi, num_examples)\n",
    "print(twi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OmMZQpdO60dt"
   },
   "outputs": [],
   "source": [
    "def max_length(tensor):\n",
    "  return max(len(t) for t in tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bIOn8RCNDJXG"
   },
   "outputs": [],
   "source": [
    "def tokenize(lang):\n",
    "  lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(\n",
    "      filters='')\n",
    "  lang_tokenizer.fit_on_texts(lang)\n",
    "\n",
    "  tensor = lang_tokenizer.texts_to_sequences(lang)\n",
    "\n",
    "  tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor,\n",
    "                                                         padding='post')\n",
    "\n",
    "  return tensor, lang_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VAMb7gF8eOf5"
   },
   "outputs": [],
   "source": [
    "def load_data( num_examples=None):\n",
    "  # creating cleaned input, output pairs\n",
    "  targ_lang  = english_d[0:num_examples]\n",
    "  inp_lang =   twi_d[0:num_examples]\n",
    "\n",
    "  input_tensor, inp_lang_tokenizer = tokenize(inp_lang)\n",
    "  target_tensor, targ_lang_tokenizer = tokenize(targ_lang)\n",
    "\n",
    "  return input_tensor, target_tensor, inp_lang_tokenizer, targ_lang_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ld1b5ps7yy9L"
   },
   "outputs": [],
   "source": [
    "# Try experimenting with the size of that dataset\n",
    "num_examples = 3000\n",
    "input_tensor, target_tensor, inp_lang, targ_lang = load_data( num_examples)\n",
    "\n",
    "# Calculate max_length of the target tensors\n",
    "max_length_targ, max_length_inp = max_length(target_tensor), max_length(input_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "id": "tQKYLwl0Gh5V",
    "outputId": "6fc7a40e-7997-4407-b358-ccbcaa7f65dd"
   },
   "outputs": [],
   "source": [
    "print(max_length_targ)\n",
    "print(max_length_inp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Ary6bwnA4dQc",
    "outputId": "2b68b0df-6b15-4075-d46a-016f21709f30"
   },
   "outputs": [],
   "source": [
    "# Creating training and validation sets using an 80-20 split\n",
    "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n",
    "\n",
    "# Show length\n",
    "print(len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lJPmLZGMeD5q"
   },
   "outputs": [],
   "source": [
    "def convert(lang, tensor):\n",
    "  for t in tensor:\n",
    "    if t!=0:\n",
    "      print (\"%d ----> %s\" % (t, lang.index_word[t]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "VXukARTDd7MT",
    "outputId": "f0a8bafb-cd44-41a8-cde8-78463c53e73d"
   },
   "outputs": [],
   "source": [
    "print (\"Input Language; index to word mapping\")\n",
    "convert(inp_lang, input_tensor_train[0])\n",
    "print ()\n",
    "print (\"Target Language; index to word mapping\")\n",
    "convert(targ_lang, target_tensor_train[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rgCLkfv5uO3d"
   },
   "source": [
    "### Create a tf.data dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TqHsArVZ3jFS"
   },
   "outputs": [],
   "source": [
    "BUFFER_SIZE = len(input_tensor_train)\n",
    "BATCH_SIZE = 64\n",
    "#BATCH_SIZE = 32\n",
    "#for step in xrange(test_size / BATCH_SIZE):\n",
    "#for step in xrange(test_size // BATCH_SIZE):\n",
    "#In order to pass a int instead of a float to xrange and avoid a runtime exception.\n",
    "steps_per_epoch = len(input_tensor_train)//BATCH_SIZE\n",
    "embedding_dim = 256\n",
    "units = 1024\n",
    "vocab_inp_size = len(inp_lang.word_index)+1\n",
    "vocab_tar_size = len(targ_lang.word_index)+1\n",
    "\n",
    "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n",
    "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "qc6-NK1GtWQt",
    "outputId": "69f8fc59-aa67-4656-e8e0-b3726900a3e1"
   },
   "outputs": [],
   "source": [
    "example_input_batch, example_target_batch = next(iter(dataset))\n",
    "example_input_batch.shape, example_target_batch.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 151
    },
    "colab_type": "code",
    "id": "n-04HrhyUE4U",
    "outputId": "6ade0816-56d7-4f38-f293-f771a996b8bf"
   },
   "outputs": [],
   "source": [
    "example_input_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nZ2rI24i3jFg"
   },
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.Model):\n",
    "  def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
    "    super(Encoder, self).__init__()\n",
    "    self.batch_sz = batch_sz\n",
    "    self.enc_units = enc_units\n",
    "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "    self.gru = tf.keras.layers.GRU(self.enc_units,\n",
    "                                   return_sequences=True,\n",
    "                                   return_state=True,\n",
    "                                   recurrent_initializer='glorot_uniform')\n",
    "\n",
    "  def call(self, x, hidden):\n",
    "    x = self.embedding(x)\n",
    "    output, state = self.gru(x, initial_state = hidden)\n",
    "    return output, state\n",
    "\n",
    "  def initialize_hidden_state(self):\n",
    "    return tf.zeros((self.batch_sz, self.enc_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "60gSVh05Jl6l",
    "outputId": "c7253635-03ff-490e-c0ad-1c3c05117fb2"
   },
   "outputs": [],
   "source": [
    "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
    "\n",
    "# sample input\n",
    "sample_hidden = encoder.initialize_hidden_state()\n",
    "sample_output, sample_hidden = encoder(example_input_batch, sample_hidden)\n",
    "print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n",
    "print ('Encoder Hidden state shape: (batch size, units) {}'.format(sample_hidden.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "umohpBN2OM94"
   },
   "outputs": [],
   "source": [
    "class BahdanauAttention(tf.keras.layers.Layer):\n",
    "  def __init__(self, units):\n",
    "    super(BahdanauAttention, self).__init__()\n",
    "    self.W1 = tf.keras.layers.Dense(units)\n",
    "    self.W2 = tf.keras.layers.Dense(units)\n",
    "    self.V = tf.keras.layers.Dense(1)\n",
    "\n",
    "  def call(self, query, values):\n",
    "    # hidden shape == (batch_size, hidden size)\n",
    "    # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n",
    "    # we are doing this to perform addition to calculate the score\n",
    "    hidden_with_time_axis = tf.expand_dims(query, 1)\n",
    "\n",
    "    # score shape == (batch_size, max_length, 1)\n",
    "    # we get 1 at the last axis because we are applying score to self.V\n",
    "    # the shape of the tensor before applying self.V is (batch_size, max_length, units)\n",
    "    score = self.V(tf.nn.tanh(\n",
    "        self.W1(values) + self.W2(hidden_with_time_axis)))\n",
    "\n",
    "    # attention_weights shape == (batch_size, max_length, 1)\n",
    "    attention_weights = tf.nn.softmax(score, axis=1)\n",
    "\n",
    "    # context_vector shape after sum == (batch_size, hidden_size)\n",
    "    context_vector = attention_weights * values\n",
    "    context_vector = tf.reduce_sum(context_vector, axis=1)\n",
    "\n",
    "    return context_vector, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "k534zTHiDjQU",
    "outputId": "db3eb0be-9785-413a-800e-f8f2df96b474"
   },
   "outputs": [],
   "source": [
    "attention_layer = BahdanauAttention(10)\n",
    "attention_result, attention_weights = attention_layer(sample_hidden, sample_output)\n",
    "\n",
    "print(\"Attention result shape: (batch size, units) {}\".format(attention_result.shape))\n",
    "print(\"Attention weights shape: (batch_size, sequence_length, 1) {}\".format(attention_weights.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yJ_B3mhW3jFk"
   },
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.Model):\n",
    "  def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
    "    super(Decoder, self).__init__()\n",
    "    self.batch_sz = batch_sz\n",
    "    self.dec_units = dec_units\n",
    "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "    self.gru = tf.keras.layers.GRU(self.dec_units,\n",
    "                                   return_sequences=True,\n",
    "                                   return_state=True,\n",
    "                                   recurrent_initializer='glorot_uniform')\n",
    "    self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "\n",
    "    # used for attention\n",
    "    self.attention = BahdanauAttention(self.dec_units)\n",
    "\n",
    "  def call(self, x, hidden, enc_output):\n",
    "    # enc_output shape == (batch_size, max_length, hidden_size)\n",
    "    context_vector, attention_weights = self.attention(hidden, enc_output)\n",
    "\n",
    "    # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
    "    x = self.embedding(x)\n",
    "\n",
    "    # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
    "    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
    "\n",
    "    # passing the concatenated vector to the GRU\n",
    "    output, state = self.gru(x)\n",
    "\n",
    "    # output shape == (batch_size * 1, hidden_size)\n",
    "    output = tf.reshape(output, (-1, output.shape[2]))\n",
    "\n",
    "    # output shape == (batch_size, vocab)\n",
    "    x = self.fc(output)\n",
    "\n",
    "    return x, state, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "P5UY8wko3jFp",
    "outputId": "cdd68ac1-b06b-4285-af06-fede682b1267"
   },
   "outputs": [],
   "source": [
    "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n",
    "\n",
    "sample_decoder_output, _, _ = decoder(tf.random.uniform((BATCH_SIZE, 1)),\n",
    "                                      sample_hidden, sample_output)\n",
    "\n",
    "print ('Decoder output shape: (batch_size, vocab size) {}'.format(sample_decoder_output.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_ch_71VbIRfK"
   },
   "source": [
    "## Define the optimizer and the loss function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WmTHr5iV3jFr"
   },
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.Adam()\n",
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
    "    from_logits=True, reduction='none')\n",
    "\n",
    "def loss_function(real, pred):\n",
    "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
    "  loss_ = loss_object(real, pred)\n",
    "\n",
    "  mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "  loss_ *= mask\n",
    "\n",
    "  return tf.reduce_mean(loss_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DMVWzzsfNl4e"
   },
   "source": [
    "## Checkpoints (Object-based saving)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Zj8bXQTgNwrF"
   },
   "outputs": [],
   "source": [
    "checkpoint_dir = './training_checkpoints'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                                 encoder=encoder,\n",
    "                                 decoder=decoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sC9ArXSsVfqn"
   },
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def train_step(inp, targ, enc_hidden):\n",
    "  loss = 0\n",
    "\n",
    "  with tf.GradientTape() as tape:\n",
    "    enc_output, enc_hidden = encoder(inp, enc_hidden)\n",
    "\n",
    "    dec_hidden = enc_hidden\n",
    "\n",
    "    dec_input = tf.expand_dims([targ_lang.word_index['<start>']] * BATCH_SIZE, 1)\n",
    "\n",
    "    # Teacher forcing - feeding the target as the next input\n",
    "    for t in range(1, targ.shape[1]):\n",
    "      # passing enc_output to the decoder\n",
    "      predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
    "\n",
    "      loss += loss_function(targ[:, t], predictions)\n",
    "\n",
    "      # using teacher forcing\n",
    "      dec_input = tf.expand_dims(targ[:, t], 1)\n",
    "\n",
    "  batch_loss = (loss / int(targ.shape[1]))\n",
    "\n",
    "  variables = encoder.trainable_variables + decoder.trainable_variables\n",
    "\n",
    "  gradients = tape.gradient(loss, variables)\n",
    "\n",
    "  optimizer.apply_gradients(zip(gradients, variables))\n",
    "\n",
    "  return batch_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 697
    },
    "colab_type": "code",
    "id": "ddefjBMa3jF0",
    "outputId": "fa7173f0-39af-4794-94fe-4389ca7e8a15"
   },
   "outputs": [],
   "source": [
    "EPOCHS = 10\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "  start = time.time()\n",
    "\n",
    "  enc_hidden = encoder.initialize_hidden_state()\n",
    "  total_loss = 0\n",
    "\n",
    "  for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):\n",
    "    batch_loss = train_step(inp, targ, enc_hidden)\n",
    "    total_loss += batch_loss\n",
    "\n",
    "    if batch % 100 == 0:\n",
    "      print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                                   batch,\n",
    "                                                   batch_loss.numpy()))\n",
    "  # saving (checkpoint) the model every 2 epochs\n",
    "  if (epoch + 1) % 2 == 0:\n",
    "    checkpoint.save(file_prefix = checkpoint_prefix)\n",
    "\n",
    "  print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                      total_loss / steps_per_epoch))\n",
    "  print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EPOCHS = 10\n",
    "\n",
    "\n",
    "with tf.Session() as sess:\n",
    "     saver = tf.train.Saver()\n",
    "     # Training of the model\n",
    "    for epoch in range(EPOCHS):\n",
    "       start = time.time()\n",
    "\n",
    "       enc_hidden = encoder.initialize_hidden_state()\n",
    "       total_loss = 0\n",
    "\n",
    "        for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):\n",
    "           batch_loss = train_step(inp, targ, enc_hidden)\n",
    "           total_loss += batch_loss\n",
    "\n",
    "          if batch % 100 == 0:\n",
    "           print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                                   batch,\n",
    "                                                   batch_loss.numpy()))\n",
    "         # saving (checkpoint) the model every 2 epochs\n",
    "        if (epoch + 1) % 2 == 0:\n",
    "          checkpoint.save(file_prefix = checkpoint_prefix)\n",
    "\n",
    "         print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                      total_loss / steps_per_epoch))\n",
    "         print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))\n",
    "    \n",
    "         save_path = saver.save(sess, \"/home/nanaboateng/notebooks/NLP/encoder_decoder\")\n",
    "        print(f\"Model saved in path {save_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "     saver = tf.train.Saver()\n",
    "     # Training of the model\n",
    "     \n",
    "     save_path = saver.save(sess, \"/home/nanaboateng/notebooks/Data/encoder_decoder\")\n",
    "     print(f\"Model saved in path {save_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EbQpyYs13jF_"
   },
   "outputs": [],
   "source": [
    "def evaluate(sentence):\n",
    "  attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
    "\n",
    "  sentence = preprocess_sentence_twi(sentence)\n",
    "\n",
    "  inputs = [inp_lang.word_index[i] for i in sentence.split(' ')]\n",
    "  inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],\n",
    "                                                         maxlen=max_length_inp,\n",
    "                                                         padding='post')\n",
    "  inputs = tf.convert_to_tensor(inputs)\n",
    "\n",
    "  result = ''\n",
    "\n",
    "  hidden = [tf.zeros((1, units))]\n",
    "  enc_out, enc_hidden = encoder(inputs, hidden)\n",
    "\n",
    "  dec_hidden = enc_hidden\n",
    "  dec_input = tf.expand_dims([targ_lang.word_index['<start>']], 0)\n",
    "\n",
    "  for t in range(max_length_targ):\n",
    "    predictions, dec_hidden, attention_weights = decoder(dec_input,\n",
    "                                                         dec_hidden,\n",
    "                                                         enc_out)\n",
    "\n",
    "    # storing the attention weights to plot later on\n",
    "    attention_weights = tf.reshape(attention_weights, (-1, ))\n",
    "    attention_plot[t] = attention_weights.numpy()\n",
    "\n",
    "    predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "\n",
    "    result += targ_lang.index_word[predicted_id] + ' '\n",
    "\n",
    "    if targ_lang.index_word[predicted_id] == '<end>':\n",
    "      return result, sentence, attention_plot\n",
    "\n",
    "    # the predicted ID is fed back into the model\n",
    "    dec_input = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "  return result, sentence, attention_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "s5hQWlbN3jGF"
   },
   "outputs": [],
   "source": [
    "# function for plotting the attention weights\n",
    "def plot_attention(attention, sentence, predicted_sentence):\n",
    "  fig = plt.figure(figsize=(10,10))\n",
    "  ax = fig.add_subplot(1, 1, 1)\n",
    "  ax.matshow(attention, cmap='viridis')\n",
    "\n",
    "  fontdict = {'fontsize': 14}\n",
    "\n",
    "  ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
    "  ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
    "\n",
    "  ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "  ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "\n",
    "  plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "sl9zUHzg3jGI"
   },
   "outputs": [],
   "source": [
    "def translate(sentence):\n",
    "  result, sentence, attention_plot = evaluate(sentence)\n",
    "\n",
    "  print('Input: %s' % (sentence))\n",
    "  print('Predicted translation: {}'.format(result))\n",
    "\n",
    "  attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
    "  plot_attention(attention_plot, sentence.split(' '), result.split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "UJpT9D5_OgP6",
    "outputId": "546f4fa6-4bf2-4ba4-bedd-9743ac9c9708"
   },
   "outputs": [],
   "source": [
    "# restoring the latest checkpoint in checkpoint_dir\n",
    "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 620
    },
    "colab_type": "code",
    "id": "WrAM0FDomq3E",
    "outputId": "c5f43b12-3851-47a9-a0d4-f86b7cd74341"
   },
   "outputs": [],
   "source": [
    "#translate(u'hace mucho frio aqui.')\n",
    "translate(u'kristo nkutoo ne yɛn gyefo .')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "TPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "English_Twi_Neural_Machine_Translation_with_attention_3.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "py37_default",
   "language": "python",
   "name": "conda-env-py37_default-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}