code_translation.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SV9K-SohUL8m"
      },
      "source": [
        "# Machine Translation: Russian-to-English & English-to-Russian\n",
        "\n",
        "This project utilizes RNNs as well as a pre-trained GloVe embeddings for the purposes of machine translation."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xu0dursrUIRo"
      },
      "outputs": [],
      "source": [
        "# importing necessary libraries\n",
        "from __future__ import unicode_literals, print_function, division\n",
        "from io import open\n",
        "import unicodedata\n",
        "import string\n",
        "import re\n",
        "import random\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from torch import optim\n",
        "import torch.nn.functional as F\n",
        "\n",
        "import time\n",
        "import math\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "plt.switch_backend('agg')\n",
        "import matplotlib.ticker as ticker\n",
        "import numpy as np\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tnzgl_maVzm7"
      },
      "source": [
        "## Model 1: Russian-to-English\n",
        "\n",
        "Base code was taken from Author: Sean Robertson <https://github.com/spro/practical-pytorch>_"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6jylmZQrVsyP"
      },
      "outputs": [],
      "source": [
        "# to create a unique index per word\n",
        "# SOS: start of sentence\n",
        "# EOS: end of sentence\n",
        "\n",
        "SOS_token = 0\n",
        "EOS_token = 1\n",
        "\n",
        "\n",
        "class Lang:\n",
        "    def __init__(self, name):\n",
        "        self.name = name\n",
        "        self.word2index = {}\n",
        "        self.word2count = {}\n",
        "        self.index2word = {0: \"SOS\", 1: \"EOS\"}\n",
        "        self.n_words = 2  # Count SOS and EOS\n",
        "\n",
        "    def addSentence(self, sentence):\n",
        "        for word in sentence.split(' '):\n",
        "            self.addWord(word)\n",
        "\n",
        "    def addWord(self, word):\n",
        "        if word not in self.word2index:\n",
        "            self.word2index[word] = self.n_words\n",
        "            self.word2count[word] = 1\n",
        "            self.index2word[self.n_words] = word\n",
        "            self.n_words += 1\n",
        "        else:\n",
        "            self.word2count[word] += 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "TVQKzgW1WLMm"
      },
      "outputs": [],
      "source": [
        "# turn a Unicode string to plain ASCII\n",
        "# https://stackoverflow.com/a/518232/2809427\n",
        "\n",
        "def unicodeToAscii(s):\n",
        "    return ''.join(\n",
        "        c for c in unicodedata.normalize('NFD', s)\n",
        "        if unicodedata.category(c) != 'Mn'\n",
        "    )\n",
        "\n",
        "# make lowercase, trim, and remove non-letter characters\n",
        "def normalizeString(s):\n",
        "    s = unicodeToAscii(s.lower().strip())\n",
        "    s = re.sub(r\"([.!?])\", r\" \\1\", s)\n",
        "    s = re.sub(r\"[^a-zA-Zа-яА-Я.!?]+\", r\" \", s) # added cyrillic letters\n",
        "    return s"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Tiw-TzHiWVYG"
      },
      "outputs": [],
      "source": [
        "def readLangs(lang1, lang2, reverse=False):\n",
        "    print(\"Reading lines...\")\n",
        "\n",
        "    # Read the file and split into lines\n",
        "    lines = open('/content/%s-%s.txt' % (lang1, lang2), encoding='utf-8').\\\n",
        "        read().strip().split('\\n')\n",
        "\n",
        "    # Split every line into pairs and normalize\n",
        "    pairs = [[normalizeString(s) for s in l.split('\\t')[:2]] for l in lines] # added a [:2] since there were\n",
        "    #copywrite notes on each line\n",
        "\n",
        "    # Reverse pairs if needed\n",
        "    if reverse:\n",
        "        pairs = [list(reversed(p)) for p in pairs]\n",
        "        input_lang = Lang(lang2)\n",
        "        output_lang = Lang(lang1)\n",
        "    else:\n",
        "        input_lang = Lang(lang1)\n",
        "        output_lang = Lang(lang2)\n",
        "\n",
        "    return input_lang, output_lang, pairs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "r7oHNWWsW456"
      },
      "outputs": [],
      "source": [
        "# trim the data since there are a lot of examples and we don't want this to take forever\n",
        "MAX_LENGTH = 10\n",
        "\n",
        "eng_prefixes = (\n",
        "    \"i am \", \"i'm \",\n",
        "    \"he is\", \"he's \",\n",
        "    \"she is\", \"she's \",\n",
        "    \"you are\", \"you're \",\n",
        "    \"we are\", \"we're \",\n",
        "    \"they are\", \"they're \"\n",
        ")\n",
        "\n",
        "\n",
        "# filters\n",
        "def filterPair(p):\n",
        "    return len(p[0].split(' ')) < MAX_LENGTH and \\\n",
        "        len(p[1].split(' ')) < MAX_LENGTH and \\\n",
        "        p[1].startswith(eng_prefixes)\n",
        "\n",
        "\n",
        "def filterPairs(pairs):\n",
        "    return [pair for pair in pairs if filterPair(pair)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Czrpi-h-XdCo",
        "outputId": "d4c85513-866f-4dfb-bcc9-96a8e4393176"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading lines...\n",
            "Read 479223 sentence pairs\n",
            "Trimmed to 3879 sentence pairs\n",
            "Counting words...\n",
            "Counted words:\n",
            "rus 3309\n",
            "eng 1735\n",
            "['ты сел не на тот поезд .', 'you are on the wrong train .']\n"
          ]
        }
      ],
      "source": [
        "# full data preparation function:\n",
        "def prepareData(lang1, lang2, reverse=False):\n",
        "    input_lang, output_lang, pairs = readLangs(lang1, lang2, reverse) #read the data\n",
        "    print(\"Read %s sentence pairs\" % len(pairs))\n",
        "    pairs = filterPairs(pairs) # filter and trim\n",
        "    print(\"Trimmed to %s sentence pairs\" % len(pairs))\n",
        "    print(\"Counting words...\")\n",
        "    for pair in pairs:\n",
        "        input_lang.addSentence(pair[0]) # make pairs into lists\n",
        "        output_lang.addSentence(pair[1])\n",
        "    print(\"Counted words:\")\n",
        "    print(input_lang.name, input_lang.n_words)\n",
        "    print(output_lang.name, output_lang.n_words)\n",
        "    return input_lang, output_lang, pairs\n",
        "\n",
        "\n",
        "input_lang, output_lang, pairs = prepareData('eng', 'rus', True)\n",
        "print(random.choice(pairs))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "716jcT9RZ3n7"
      },
      "outputs": [],
      "source": [
        "#ENCODER - outputs some value for every word from the input sentence.\n",
        "#For every input word the encoder outputs a\n",
        "#vector and a hidden state, and uses the hidden state for the next input word.\n",
        "class EncoderRNN(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size):\n",
        "        super(EncoderRNN, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "\n",
        "        self.embedding = nn.Embedding(input_size, hidden_size)\n",
        "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
        "\n",
        "    def forward(self, input, hidden):\n",
        "        embedded = self.embedding(input).view(1, 1, -1)\n",
        "        output = embedded\n",
        "        output, hidden = self.gru(output, hidden)\n",
        "        return output, hidden\n",
        "\n",
        "    def initHidden(self):\n",
        "        return torch.zeros(1, 1, self.hidden_size, device=device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uk4Do8O8dlu5"
      },
      "outputs": [],
      "source": [
        "#DECODER - takes the encoder output vector(s) and outputs a sequence of words to create the translation.\n",
        "# using attention decoder which focuses on parts of outputs\n",
        "class AttnDecoderRNN(nn.Module):\n",
        "    def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=MAX_LENGTH):\n",
        "        super(AttnDecoderRNN, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.output_size = output_size\n",
        "        self.dropout_p = dropout_p\n",
        "        self.max_length = max_length\n",
        "\n",
        "        self.embedding = nn.Embedding(self.output_size, self.hidden_size)\n",
        "        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)\n",
        "        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)\n",
        "        self.dropout = nn.Dropout(self.dropout_p)\n",
        "        self.gru = nn.GRU(self.hidden_size, self.hidden_size)\n",
        "        self.out = nn.Linear(self.hidden_size, self.output_size)\n",
        "\n",
        "    def forward(self, input, hidden, encoder_outputs):\n",
        "        embedded = self.embedding(input).view(1, 1, -1)\n",
        "        embedded = self.dropout(embedded)\n",
        "\n",
        "        attn_weights = F.softmax(\n",
        "            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)\n",
        "        attn_applied = torch.bmm(attn_weights.unsqueeze(0),\n",
        "                                 encoder_outputs.unsqueeze(0))\n",
        "\n",
        "        output = torch.cat((embedded[0], attn_applied[0]), 1)\n",
        "        output = self.attn_combine(output).unsqueeze(0)\n",
        "\n",
        "        output = F.relu(output)\n",
        "        output, hidden = self.gru(output, hidden)\n",
        "\n",
        "        output = F.log_softmax(self.out(output[0]), dim=1)\n",
        "        return output, hidden, attn_weights\n",
        "\n",
        "    def initHidden(self):\n",
        "        return torch.zeros(1, 1, self.hidden_size, device=device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HisTLLP1hHUn"
      },
      "outputs": [],
      "source": [
        "# TRAINING prep - adding indexes and marking EOS\n",
        "def indexesFromSentence(lang, sentence):\n",
        "    return [lang.word2index[word] for word in sentence.split(' ')]\n",
        "\n",
        "\n",
        "def tensorFromSentence(lang, sentence):\n",
        "    indexes = indexesFromSentence(lang, sentence)\n",
        "    indexes.append(EOS_token)\n",
        "    return torch.tensor(indexes, dtype=torch.long, device=device).view(-1, 1)\n",
        "\n",
        "\n",
        "def tensorsFromPair(pair):\n",
        "    input_tensor = tensorFromSentence(input_lang, pair[0])\n",
        "    target_tensor = tensorFromSentence(output_lang, pair[1])\n",
        "    return (input_tensor, target_tensor)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QjUqt2wfhUkQ"
      },
      "outputs": [],
      "source": [
        "# TRAIN function - randomly if are gonna use\n",
        "teacher_forcing_ratio = 0.5\n",
        "\n",
        "def train(input_tensor, target_tensor, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, max_length=MAX_LENGTH):\n",
        "    encoder_hidden = encoder.initHidden()\n",
        "\n",
        "    encoder_optimizer.zero_grad()\n",
        "    decoder_optimizer.zero_grad()\n",
        "\n",
        "    input_length = input_tensor.size(0)\n",
        "    target_length = target_tensor.size(0)\n",
        "\n",
        "    encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
        "\n",
        "    loss = 0\n",
        "\n",
        "    for ei in range(input_length):\n",
        "        encoder_output, encoder_hidden = encoder(\n",
        "            input_tensor[ei], encoder_hidden)\n",
        "        encoder_outputs[ei] = encoder_output[0, 0]\n",
        "\n",
        "    decoder_input = torch.tensor([[SOS_token]], device=device)\n",
        "\n",
        "    decoder_hidden = encoder_hidden\n",
        "\n",
        "    use_teacher_forcing = True if random.random() < teacher_forcing_ratio else False\n",
        "\n",
        "    if use_teacher_forcing:\n",
        "        # Teacher forcing: Feed the target as the next input\n",
        "        for di in range(target_length):\n",
        "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
        "                decoder_input, decoder_hidden, encoder_outputs)\n",
        "            loss += criterion(decoder_output, target_tensor[di])\n",
        "            decoder_input = target_tensor[di]  # Teacher forcing\n",
        "\n",
        "    else:\n",
        "        # Without teacher forcing: use its own predictions as the next input\n",
        "        for di in range(target_length):\n",
        "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
        "                decoder_input, decoder_hidden, encoder_outputs)\n",
        "            topv, topi = decoder_output.topk(1)\n",
        "            decoder_input = topi.squeeze().detach()  # detach from history as input\n",
        "\n",
        "            loss += criterion(decoder_output, target_tensor[di])\n",
        "            if decoder_input.item() == EOS_token:\n",
        "                break\n",
        "\n",
        "    loss.backward()\n",
        "\n",
        "    encoder_optimizer.step()\n",
        "    decoder_optimizer.step()\n",
        "\n",
        "    return loss.item() / target_length"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eviMdEFhh2tH"
      },
      "outputs": [],
      "source": [
        "# to keep track of evapsed time\n",
        "def asMinutes(s):\n",
        "    m = math.floor(s / 60)\n",
        "    s -= m * 60\n",
        "    return '%dm %ds' % (m, s)\n",
        "\n",
        "\n",
        "def timeSince(since, percent):\n",
        "    now = time.time()\n",
        "    s = now - since\n",
        "    es = s / (percent)\n",
        "    rs = es - s\n",
        "    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))\n",
        "\n",
        "#plotting results\n",
        "def showPlot(points):\n",
        "    plt.figure()\n",
        "    fig, ax = plt.subplots()\n",
        "    # this locator puts ticks at regular intervals\n",
        "    loc = ticker.MultipleLocator(base=0.2)\n",
        "    ax.yaxis.set_major_locator(loc)\n",
        "    plt.plot(points)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LQo6Kcwzh915"
      },
      "outputs": [],
      "source": [
        "# ITERACTIONS OF TRAIN\n",
        "def trainIters(encoder, decoder, n_iters, print_every=1000, plot_every=100, learning_rate=0.01):\n",
        "    start = time.time()\n",
        "    plot_losses = []\n",
        "    print_loss_total = 0  # Reset every print_every\n",
        "    plot_loss_total = 0  # Reset every plot_every\n",
        "\n",
        "    encoder_optimizer = optim.SGD(encoder.parameters(), lr=learning_rate)\n",
        "    decoder_optimizer = optim.SGD(decoder.parameters(), lr=learning_rate)\n",
        "    training_pairs = [tensorsFromPair(random.choice(pairs))\n",
        "                      for i in range(n_iters)]\n",
        "    criterion = nn.NLLLoss()\n",
        "\n",
        "    for iter in range(1, n_iters + 1):\n",
        "        training_pair = training_pairs[iter - 1]\n",
        "        input_tensor = training_pair[0]\n",
        "        target_tensor = training_pair[1]\n",
        "\n",
        "        loss = train(input_tensor, target_tensor, encoder,\n",
        "                     decoder, encoder_optimizer, decoder_optimizer, criterion)\n",
        "        print_loss_total += loss\n",
        "        plot_loss_total += loss\n",
        "\n",
        "        if iter % print_every == 0:\n",
        "            print_loss_avg = print_loss_total / print_every\n",
        "            print_loss_total = 0\n",
        "            print('%s (%d %d%%) %.4f' % (timeSince(start, iter / n_iters),\n",
        "                                         iter, iter / n_iters * 100, print_loss_avg))\n",
        "\n",
        "        if iter % plot_every == 0:\n",
        "            plot_loss_avg = plot_loss_total / plot_every\n",
        "            plot_losses.append(plot_loss_avg)\n",
        "            plot_loss_total = 0\n",
        "\n",
        "    showPlot(plot_losses)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vcEgKC8DieNf"
      },
      "outputs": [],
      "source": [
        "# EVALUATION (similar to training but no targets - just decoder output)\n",
        "def evaluate(encoder, decoder, sentence, max_length=MAX_LENGTH):\n",
        "    with torch.no_grad():\n",
        "        input_tensor = tensorFromSentence(input_lang, sentence)\n",
        "        input_length = input_tensor.size()[0]\n",
        "        encoder_hidden = encoder.initHidden()\n",
        "\n",
        "        encoder_outputs = torch.zeros(max_length, encoder.hidden_size, device=device)\n",
        "\n",
        "        for ei in range(input_length):\n",
        "            encoder_output, encoder_hidden = encoder(input_tensor[ei],\n",
        "                                                     encoder_hidden)\n",
        "            encoder_outputs[ei] += encoder_output[0, 0]\n",
        "\n",
        "        decoder_input = torch.tensor([[SOS_token]], device=device)  # SOS\n",
        "\n",
        "        decoder_hidden = encoder_hidden\n",
        "\n",
        "        decoded_words = []\n",
        "        decoder_attentions = torch.zeros(max_length, max_length)\n",
        "\n",
        "        for di in range(max_length):\n",
        "            decoder_output, decoder_hidden, decoder_attention = decoder(\n",
        "                decoder_input, decoder_hidden, encoder_outputs)\n",
        "            decoder_attentions[di] = decoder_attention.data\n",
        "            topv, topi = decoder_output.data.topk(1)\n",
        "            if topi.item() == EOS_token:\n",
        "                decoded_words.append('<EOS>')\n",
        "                break\n",
        "            else:\n",
        "                decoded_words.append(output_lang.index2word[topi.item()])\n",
        "\n",
        "            decoder_input = topi.squeeze().detach()\n",
        "\n",
        "        return decoded_words, decoder_attentions[:di + 1]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hr-fvhx3il83"
      },
      "outputs": [],
      "source": [
        "# random evaluation\n",
        "def evaluateRandomly(encoder, decoder, n=10):\n",
        "    for i in range(n):\n",
        "        pair = random.choice(pairs)\n",
        "        print('>', pair[0])\n",
        "        print('=', pair[1])\n",
        "        output_words, attentions = evaluate(encoder, decoder, pair[0])\n",
        "        output_sentence = ' '.join(output_words)\n",
        "        print('<', output_sentence)\n",
        "        print('')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c6PEeIw1jRSV",
        "outputId": "e6ec0a71-b3f3-4912-cb9f-639d661cee1c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1m 28s (- 20m 38s) (5000 6%) 2.8988\n",
            "2m 51s (- 18m 37s) (10000 13%) 2.1663\n",
            "4m 20s (- 17m 22s) (15000 20%) 1.6673\n",
            "5m 46s (- 15m 52s) (20000 26%) 1.2768\n",
            "7m 12s (- 14m 24s) (25000 33%) 0.9662\n",
            "8m 37s (- 12m 56s) (30000 40%) 0.7064\n",
            "10m 3s (- 11m 30s) (35000 46%) 0.5060\n",
            "11m 28s (- 10m 2s) (40000 53%) 0.3697\n",
            "12m 53s (- 8m 35s) (45000 60%) 0.2466\n",
            "14m 18s (- 7m 9s) (50000 66%) 0.1874\n",
            "15m 43s (- 5m 43s) (55000 73%) 0.1184\n",
            "17m 7s (- 4m 16s) (60000 80%) 0.0844\n",
            "18m 32s (- 2m 51s) (65000 86%) 0.0611\n",
            "19m 57s (- 1m 25s) (70000 93%) 0.0549\n",
            "21m 21s (- 0m 0s) (75000 100%) 0.0588\n"
          ]
        }
      ],
      "source": [
        "# TRAIN AND EVALUATE MODEL 1\n",
        "hidden_size = 256\n",
        "encoder1 = EncoderRNN(input_lang.n_words, hidden_size).to(device)\n",
        "attn_decoder1 = AttnDecoderRNN(hidden_size, output_lang.n_words, dropout_p=0.1).to(device)\n",
        "\n",
        "trainIters(encoder1, attn_decoder1, 75000, print_every=5000)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iHXXDpKvjzOL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "64548c96-4e1d-4ff9-fd94-4325798d3b51"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "> я объясняю правила .\n",
            "= i am explaining the rules .\n",
            "< i am explaining the rules . <EOS>\n",
            "\n",
            "> сеичас у него дела лучше чем раньше .\n",
            "= he is better off than he was .\n",
            "< he is better off than he was . <EOS>\n",
            "\n",
            "> он влиятелен .\n",
            "= he is influential .\n",
            "< he is powerful . <EOS>\n",
            "\n",
            "> у нее нет стыда .\n",
            "= she is shameless .\n",
            "< she is shameless . <EOS>\n",
            "\n",
            "> он пытается совмещать две работы .\n",
            "= he is trying to maintain two jobs .\n",
            "< he is trying to maintain two jobs . <EOS>\n",
            "\n",
            "> он сегодня дома .\n",
            "= he is at home today .\n",
            "< he is at home today . <EOS>\n",
            "\n",
            "> он красивыи .\n",
            "= he is handsome .\n",
            "< he is handsome . <EOS>\n",
            "\n",
            "> боюсь ваш план не будет работать .\n",
            "= i am afraid your plan will not work .\n",
            "< i am afraid your plan will not work . <EOS>\n",
            "\n",
            "> боюсь я не могу тебе помочь .\n",
            "= i am afraid i can t help you .\n",
            "< i am afraid i can t help you . <EOS>\n",
            "\n",
            "> он слишком мал чтобы купаться одному .\n",
            "= he is too young to go swimming alone .\n",
            "< he is too young to go swimming alone . <EOS>\n",
            "\n"
          ]
        }
      ],
      "source": [
        "evaluateRandomly(encoder1, attn_decoder1)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cWqoS8DisqTS"
      },
      "source": [
        "## Model 2: English-to-Russian with pretrained embeddings"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# had to change this function since the order is changed\n",
        "def filterPair(p):\n",
        "    return len(p[1].split(' ')) < MAX_LENGTH and \\\n",
        "        len(p[0].split(' ')) < MAX_LENGTH and \\\n",
        "        p[0].startswith(eng_prefixes)\n",
        "\n",
        "input_lang, output_lang, pairs = prepareData('eng', 'rus') # not reversed\n",
        "print(random.choice(pairs))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z1ny7nf_Q34Y",
        "outputId": "dbdeac6f-ad24-4080-fe44-1b539bc6dd09"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading lines...\n",
            "Read 479223 sentence pairs\n",
            "Trimmed to 3879 sentence pairs\n",
            "Counting words...\n",
            "Counted words:\n",
            "eng 1735\n",
            "rus 3309\n",
            "['you aren t any better than me .', 'ты ничуть не лучше меня .']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# installing GloVe\n",
        "\n",
        "#!python -m spacy download en_core_web_lg"
      ],
      "metadata": {
        "id": "c26CeENRUNuh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Pan3CB8gsuMf"
      },
      "outputs": [],
      "source": [
        "import spacy\n",
        "\n",
        "# load spaCy's pre-trained GloVe model (glove.6B.100d)\n",
        "nlp = spacy.load(\"en_core_web_lg\")\n",
        "\n",
        "# vocabulary size\n",
        "vocab_size = len(nlp.vocab.vectors)\n",
        "\n",
        "#  list to store the GloVe vectors\n",
        "glove_vectors = []\n",
        "\n",
        "for word in nlp.vocab.vectors.keys():\n",
        "    vector = nlp.vocab.vectors[word]\n",
        "    glove_vectors.append(vector)\n",
        "\n",
        "# convert to a PyTorch tensor\n",
        "pretrained_embeddings = torch.tensor(glove_vectors)\n",
        "\n",
        "class EncoderRNN(nn.Module):\n",
        "    def __init__(self, input_size, hidden_size, pretrained_embeddings=None, freeze_embeddings=True):\n",
        "        super(EncoderRNN, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "\n",
        "        if pretrained_embeddings is not None:\n",
        "            # initialize the embedding layer with pre-trained embeddings\n",
        "            self.embedding = nn.Embedding.from_pretrained(pretrained_embeddings, freeze=freeze_embeddings)\n",
        "        else:\n",
        "            # if no pre-trained embeddings are provided, create a new embedding layer\n",
        "            self.embedding = nn.Embedding(input_size, hidden_size)\n",
        "\n",
        "        self.gru = nn.GRU(hidden_size, hidden_size)\n",
        "\n",
        "    def forward(self, input, hidden):\n",
        "        embedded = self.embedding(input).view(1, 1, -1)\n",
        "        output = embedded\n",
        "        output, hidden = self.gru(output, hidden)\n",
        "        return output, hidden\n",
        "\n",
        "    def initHidden(self):\n",
        "        return torch.zeros(1, 1, self.hidden_size, device=device)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "pretrained_embeddings.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "imzxzheva3rq",
        "outputId": "47bb22ae-0b80-49bd-c432-b820953c6477"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([514157, 300])"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# decoder - same\n",
        "class AttnDecoderRNN(nn.Module):\n",
        "    def __init__(self, hidden_size, output_size, dropout_p=0.1, max_length=MAX_LENGTH):\n",
        "        super(AttnDecoderRNN, self).__init__()\n",
        "        self.hidden_size = hidden_size\n",
        "        self.output_size = output_size\n",
        "        self.dropout_p = dropout_p\n",
        "        self.max_length = max_length\n",
        "\n",
        "        self.embedding = nn.Embedding(self.output_size, self.hidden_size)\n",
        "        self.attn = nn.Linear(self.hidden_size * 2, self.max_length)\n",
        "        self.attn_combine = nn.Linear(self.hidden_size * 2, self.hidden_size)\n",
        "        self.dropout = nn.Dropout(self.dropout_p)\n",
        "        self.gru = nn.GRU(self.hidden_size, self.hidden_size)\n",
        "        self.out = nn.Linear(self.hidden_size, self.output_size)\n",
        "\n",
        "    def forward(self, input, hidden, encoder_outputs):\n",
        "        embedded = self.embedding(input).view(1, 1, -1)\n",
        "        embedded = self.dropout(embedded)\n",
        "\n",
        "        attn_weights = F.softmax(\n",
        "            self.attn(torch.cat((embedded[0], hidden[0]), 1)), dim=1)\n",
        "        attn_applied = torch.bmm(attn_weights.unsqueeze(0),\n",
        "                                 encoder_outputs.unsqueeze(0))\n",
        "\n",
        "        output = torch.cat((embedded[0], attn_applied[0]), 1)\n",
        "        output = self.attn_combine(output).unsqueeze(0)\n",
        "\n",
        "        output = F.relu(output)\n",
        "        output, hidden = self.gru(output, hidden)\n",
        "\n",
        "        output = F.log_softmax(self.out(output[0]), dim=1)\n",
        "        return output, hidden, attn_weights\n",
        "\n",
        "    def initHidden(self):\n",
        "        return torch.zeros(1, 1, self.hidden_size, device=device)"
      ],
      "metadata": {
        "id": "1Hf-LOVfSY8C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "hidden_size = 300\n",
        "# create the encoder with pre-trained embeddings\n",
        "encoder2 = EncoderRNN(vocab_size, hidden_size, pretrained_embeddings, freeze_embeddings=True).to(device)\n",
        "# decoder\n",
        "attn_decoder2 = AttnDecoderRNN(hidden_size, output_lang.n_words, dropout_p=0.1).to(device)\n",
        "# training iterations\n",
        "trainIters(encoder2, attn_decoder2, 75000, print_every=5000)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NK0UmI9HSld5",
        "outputId": "e7b0763a-6cf9-4fe9-dc26-f98bc514a2dc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1m 22s (- 19m 18s) (5000 6%) 3.5674\n",
            "2m 39s (- 17m 14s) (10000 13%) 2.7551\n",
            "3m 55s (- 15m 42s) (15000 20%) 2.1678\n",
            "5m 13s (- 14m 23s) (20000 26%) 1.7951\n",
            "6m 31s (- 13m 3s) (25000 33%) 1.4954\n",
            "7m 50s (- 11m 45s) (30000 40%) 1.2399\n",
            "9m 8s (- 10m 27s) (35000 46%) 1.0820\n",
            "10m 28s (- 9m 10s) (40000 53%) 0.9288\n",
            "11m 46s (- 7m 51s) (45000 60%) 0.8330\n",
            "13m 6s (- 6m 33s) (50000 66%) 0.7654\n",
            "14m 23s (- 5m 14s) (55000 73%) 0.6608\n",
            "15m 42s (- 3m 55s) (60000 80%) 0.5884\n",
            "17m 0s (- 2m 37s) (65000 86%) 0.5802\n",
            "18m 20s (- 1m 18s) (70000 93%) 0.5337\n",
            "19m 38s (- 0m 0s) (75000 100%) 0.4862\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluateRandomly(encoder2, attn_decoder2)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x694w_2RuB5a",
        "outputId": "0cf87d97-8ebd-4e30-eb39-4407ced31a66"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "> you aren t my mother .\n",
            "= вы мне не мать .\n",
            "< вы мне не мать . <EOS>\n",
            "\n",
            "> she is in a bad mood .\n",
            "= она в дурном настроении .\n",
            "< у нее плохое настроение . <EOS>\n",
            "\n",
            "> she is about your age .\n",
            "= она примерно вашего возраста .\n",
            "< она примерно твоего возраста . <EOS>\n",
            "\n",
            "> we are authorized to use force if necessary .\n",
            "= в случае необходимости нам разрешено применять силу .\n",
            "< в случае необходимости нам разрешено применять силу . <EOS>\n",
            "\n",
            "> she is anxious about her safety .\n",
            "= она беспокоится о ее безопасности .\n",
            "< она беспокоится о своеи безопасности . <EOS>\n",
            "\n",
            "> he isn t at home is he ?\n",
            "= его что нет дома ?\n",
            "< его ведь нет дома ? <EOS>\n",
            "\n",
            "> he is an old friend of mine .\n",
            "= он мои старыи друг .\n",
            "< он мои старыи друг . <EOS>\n",
            "\n",
            "> he is roasting coffee beans .\n",
            "= он обжаривает кофеиные зерна .\n",
            "< он обжаривает кофеиные зерна . <EOS>\n",
            "\n",
            "> i am concerned about his poor health .\n",
            "= меня беспокоит его слабое здоровье .\n",
            "< меня беспокоит его слабое здоровье . <EOS>\n",
            "\n",
            "> we aren t doing anything right now .\n",
            "= мы ничем не заняты сеичас .\n",
            "< мы сеичас ничего не делаем . <EOS>\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Input 5 well formed sentences from the English vocab to Model 2, and input the resultant translated sentences to Model 1. Display all model outputs in each case.\n"
      ],
      "metadata": {
        "id": "r2q0wBRth7uC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# evaluating second model\n",
        "def evaluateAndShowAttentionModel2(input_sentence):\n",
        "    output_words, attentions = evaluate(\n",
        "        encoder2, attn_decoder2, input_sentence)\n",
        "    print('input =', input_sentence)\n",
        "    print('output =', ' '.join(output_words))\n",
        "\n",
        "\n",
        "evaluateAndShowAttentionModel2(\"we are sorry for his mistake .\")\n",
        "\n",
        "evaluateAndShowAttentionModel2(\"you are a beautiful woman .\")\n",
        "\n",
        "evaluateAndShowAttentionModel2(\"you are too late .\")\n",
        "\n",
        "evaluateAndShowAttentionModel2(\"you aren t like the other kids .\")"
      ],
      "metadata": {
        "id": "G3zlGlsDuPRc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b4da5c61-6146-4dc6-ce84-ecfb9ff642f8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "input = we are sorry for his mistake .\n",
            "output = мы сожалеем о его ошибке . <EOS>\n",
            "input = you are a beautiful woman .\n",
            "output = ты красивая женщина . <EOS>\n",
            "input = you are too late .\n",
            "output = ты опоздал . <EOS>\n",
            "input = you aren t like the other kids .\n",
            "output = ты не такая как другие дети . <EOS>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# evaluating first model based on second model's output\n",
        "def evaluateAndShowAttentionModel1(input_sentence):\n",
        "    output_words, attentions = evaluate(\n",
        "        encoder1, attn_decoder1, input_sentence)\n",
        "    print('input =', input_sentence)\n",
        "    print('output =', ' '.join(output_words))\n",
        "\n",
        "# back to eng filters\n",
        "def filterPair(p):\n",
        "    return len(p[0].split(' ')) < MAX_LENGTH and \\\n",
        "        len(p[1].split(' ')) < MAX_LENGTH and \\\n",
        "        p[1].startswith(eng_prefixes)\n",
        "\n",
        "# reverse languages back\n",
        "input_lang, output_lang, pairs = prepareData('eng', 'rus', True)"
      ],
      "metadata": {
        "id": "HmGN2YhjiG68",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "73101089-31f3-4050-bcb5-b6481a68e930"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Reading lines...\n",
            "Read 479223 sentence pairs\n",
            "Trimmed to 3879 sentence pairs\n",
            "Counting words...\n",
            "Counted words:\n",
            "rus 3309\n",
            "eng 1735\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluateAndShowAttentionModel1(\"мы сожалеем о его ошибке .\")\n",
        "\n",
        "evaluateAndShowAttentionModel1(\"вы красивая женщина .\")\n",
        "\n",
        "evaluateAndShowAttentionModel1(\"ты опоздал .\")\n",
        "\n",
        "evaluateAndShowAttentionModel1(\"вы не такие как другие дети .\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "m664jxGHbzuP",
        "outputId": "67abf091-3c49-4e7e-f24e-a3ef7d8c1c7a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "input = мы сожалеем о его ошибке .\n",
            "output = we are sorry for his mistake . <EOS>\n",
            "input = вы красивая женщина .\n",
            "output = you are a beautiful woman . <EOS>\n",
            "input = ты опоздал .\n",
            "output = you are late . <EOS>\n",
            "input = вы не такие как другие дети .\n",
            "output = you aren t like the other kids . <EOS>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Results and Assumptions\n",
        "\n",
        "Knowing both languages, we could tell that both models perform rather well. Some differences between input and output that were observed over multiple evalutions weren't significant and still mostly remain the main meaning of the sentence. For instance, sometimes the casual word for \"you\" (ты) would be substituted by the formal version (вы) and vice-versa which isn't really a mistake.\n",
        "\n",
        "Finally, through evaluations we have noticed that the input data is not perfect: there are incomplete sentence pairs that are just cropped in the middle, for instance. This is another thing that we should keep in mind when assessing the models."
      ],
      "metadata": {
        "id": "_H8yhFfpndkH"
      }
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "gpuType": "T4",
      "authorship_tag": "ABX9TyNfvtXsidefM/PMaW9I67QW"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}