4_Transformer_Seq2Seq.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "4_Transformer_Seq2Seq.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "TZQ6ZlY2aEkg"
      },
      "source": [
        "import tensorflow_datasets as tfds\n",
        "import tensorflow as tf\n",
        "\n",
        "import time\n",
        "import numpy as np\n"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L_0sArNjsrn4",
        "outputId": "1efc5405-200f-49ce-9fe6-a533c7e3a323",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print( tf.__version__)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2.3.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-HlQpElGM-VO",
        "outputId": "d0df52c8-d67e-46bf-aebd-50f7f66725a1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "!curl -c ./cookie -s -L \"https://drive.google.com/uc?export=download&id=1LeRPXt815fFM_1jXNEIPich6D_Hf5oxB\" > /dev/null\n",
        "!curl -Lb ./cookie \"https://drive.google.com/uc?export=download&confirm=`awk '/download/ {print $NF}' ./cookie`&id=1LeRPXt815fFM_1jXNEIPich6D_Hf5oxB\" -o src"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100   408    0   408    0     0    657      0 --:--:-- --:--:-- --:--:--   657\n",
            "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
            "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
            "100 6171k    0 6171k    0     0  6789k      0 --:--:-- --:--:-- --:--:-- 6789k\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_gFVK0kkNZOr",
        "outputId": "16af8698-b32b-42d9-90b7-d5d359f21a7c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "!curl -c ./cookie -s -L \"https://drive.google.com/uc?export=download&id=1PdOCa1tl1li4QKQYtdbaqqmghVJWmkFo\" > /dev/null\n",
        "!curl -Lb ./cookie \"https://drive.google.com/uc?export=download&confirm=`awk '/download/ {print $NF}' ./cookie`&id=1PdOCa1tl1li4QKQYtdbaqqmghVJWmkFo\" -o target"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100   408    0   408    0     0    731      0 --:--:-- --:--:-- --:--:--   731\n",
            "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
            "  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\n",
            "100 5107k    0 5107k    0     0  6080k      0 --:--:-- --:--:-- --:--:-- 6080k\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NdK3yC_mbbBM"
      },
      "source": [
        "source = open('/content/src', 'r', encoding='utf-8')\n",
        "target = open('/content/target', 'r', encoding='utf-8')\n",
        "\n",
        "source_lines = source.readlines()\n",
        "target_lines = target.readlines()"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hq9RqwHZNm_o",
        "outputId": "d60bc6d8-b5b4-4e17-c154-cd4caef2e812",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        }
      },
      "source": [
        "source_lines[0:10]"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['태초 에 하나님 이 천지 를 창조 하 시 었 다 .\\n',\n",
              " '땅 이 혼돈 하 고 공허 하 며 , 어둠 이 깊 음 위 에 있 고 , 하 나 님 의 영 은 물 위 에 움직이 고 계시 었 다 .\\n',\n",
              " '하나님 이 말씀 하 시 기 를 \" 빛 이 생기 어라 \" 하 시 니 , 빛 이 생기 었 다 .\\n',\n",
              " '그 빛 이 하나님 보 시 기 에 좋 았 다 . 하나님 이 빛 과 어둠 을 나누 시 어서 ,\\n',\n",
              " '빛 을 낮 이 라고 하 시 고 , 어둠 을 밤 이 라고 하 시 었 다 . 저녁 이 되 고 아침 이 되 니 , 하루 가 지나 았 다 .\\n',\n",
              " '하나님 이 말씀 하 시 기 를 \" 물 한 가운데 창공 이 생기 어 , 물 과 물 사이 가 갈 아 지 어라 \" 하 시 었 다 .\\n',\n",
              " '하나님 이 이 처럼 창공 을 만들 시 고서 , 물 을 창공 아래 에 있 는 물 과 창공 위 에 있 는 물 로 나누 시 니 , 그대로 되 었 다 .\\n',\n",
              " '하나님 이 창공 을 하늘 이 라고 하 시 었 다 . 저녁 이 되 고 아침 이 되 니 , 이튿날 이 지나 았 다 .\\n',\n",
              " '하나님 이 말씀 하 시 기 를 \" 하늘 아래 에 있 는 물 은 한 곳 으로 모이 고 , 뭍 은 드러나 거라 \" 하 시 니 , 그대로 되 었 다 .\\n',\n",
              " '하나님 이 뭍 을 땅 이 라고 하 시 고 , 모이 ㄴ 물 을 바다 이 라고 하 시 었 다 . 하나님 보 시 기 에 좋 았 다 .\\n']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QrXh0jj_7vfS",
        "outputId": "ee01a49c-0ae4-4609-edc3-d45e9a1e5407",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 193
        }
      },
      "source": [
        "target_lines[0:10]"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['태초 에 하나님 이 천지 를 창조 하 시 니라\\n',\n",
              " '땅 이 혼돈 하 고 공허 하 며 흑 암 이 깊 음 위 에 있 고 하나님 의 영 은 수면 위 에 운행 하 시 니라\\n',\n",
              " '하나님 이 이 르시 되 빛 이 있 으라 하 시 니 빛 이 있 었 고\\n',\n",
              " '빛 이 하나님 이 보 시 기 에 좋 았 더라 하나님 이 빛 과 어둠 을 나누 사\\n',\n",
              " '하나님 이 빛 을 낮 이 라 부르 시 고 어둠 을 밤 이 라 부르 시 니라 저녁 이 되 고 아침 이 되 니 이 는 첫째 날 이 니라\\n',\n",
              " '하나님 이 이 르시 되 물 가운데 에 궁창 이 있 어 물 과 물 로 나뉘 라 하 시 고\\n',\n",
              " '하나님 이 궁창 을 만들 사 궁창 아래 의 물 과 궁창 위 의 물 로 나뉘 게 하 시 니 그대로 되 니라\\n',\n",
              " '하나님 이 궁창 을 하늘 이 라 부르 시 니라 저녁 이 되 고 아침 이 되 니 이 는 둘째 날 이 니라\\n',\n",
              " '하나님 이 이 르시 되 천하 의 물 이 한 곳 으로 모이 고 뭍 이 드러나 라 하 시 니 그대로 되 니라\\n',\n",
              " '하나님 이 뭍 을 땅 이 라 부르 시 고 모이 ㄴ 물 을 바다 라 부르 시 니 하나님 이 보 시 기 에 좋 았 더라\\n']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fu2fAdHMbslg"
      },
      "source": [
        "tokenizer_source = tfds.features.text.SubwordTextEncoder.build_from_corpus((line for line in source_lines), target_vocab_size=10000)\n",
        "tokenizer_target = tfds.features.text.SubwordTextEncoder.build_from_corpus((line for line in target_lines), target_vocab_size=10000)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cLUPZyDObydw",
        "outputId": "f0e8e520-786d-4f4f-f8ad-8a8faaa5a583",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        }
      },
      "source": [
        "text = '이순신은 조선 중기의 무신입니다.'\n",
        "\n",
        "tokenized_string = tokenizer_source.encode(text)\n",
        "print(tokenized_string)\n",
        "for ts in tokenized_string:\n",
        "    print ('{}: {}'.format(tokenizer_source.decode([ts]), ts))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[239, 2791, 1957, 13, 2060, 858, 3276, 1172, 5, 1701, 1957, 3282, 1606, 9296]\n",
            "이: 239\n",
            "순: 2791\n",
            "신: 1957\n",
            "은 : 13\n",
            "조: 2060\n",
            "선 : 858\n",
            "중: 3276\n",
            "기: 1172\n",
            "의 : 5\n",
            "무: 1701\n",
            "신: 1957\n",
            "입: 3282\n",
            "니다: 1606\n",
            ".: 9296\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jqO3EyVqeXd-"
      },
      "source": [
        "# save the vocabs\n",
        "tokenizer_source.save_to_file('/content/seq2seq_source_vocab')\n",
        "tokenizer_target.save_to_file('/content/seq2seq_target_vocab')\n",
        "\n",
        "\n",
        "# Load\n",
        "tokenizer_source = tfds.features.text.SubwordTextEncoder.load_from_file('/content/seq2seq_source_vocab')\n",
        "tokenizer_target= tfds.features.text.SubwordTextEncoder.load_from_file('/content/seq2seq_target_vocab')"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GaEa03cZcH6V"
      },
      "source": [
        "BUFFER_SIZE = 30000\n",
        "BATCH_SIZE = 64\n",
        "MAX_LENGTH = 128"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n6WLIOD5cq4f"
      },
      "source": [
        "def encode(lang1, lang2):\n",
        "    lang1 = [tokenizer_source.vocab_size] + tokenizer_source.encode(lang1.numpy()) + [tokenizer_source.vocab_size+1]\n",
        "    lang2 = [tokenizer_target.vocab_size] + tokenizer_target.encode(lang2.numpy()) + [tokenizer_target.vocab_size+1]\n",
        "    \n",
        "    return lang1, lang2"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-ausmdc1ct-H"
      },
      "source": [
        "def tf_encode(src, tar):\n",
        "    return tf.py_function(encode, [src, tar], [tf.int64, tf.int64])"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z-brUBH5czrH",
        "outputId": "200c6fe5-bb22-4697-c3b6-b7433dab4269",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        }
      },
      "source": [
        "src_dataset = tf.data.TextLineDataset('/content/src')\n",
        "target_dataset = tf.data.TextLineDataset('/content/target')\n",
        "lines_dataset = tf.data.Dataset.zip((src_dataset, target_dataset))\n",
        "\n",
        "for ex in lines_dataset.take(5):\n",
        "    print(ex)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(<tf.Tensor: shape=(), dtype=string, numpy=b'\\xed\\x83\\x9c\\xec\\xb4\\x88 \\xec\\x97\\x90 \\xed\\x95\\x98\\xeb\\x82\\x98\\xeb\\x8b\\x98 \\xec\\x9d\\xb4 \\xec\\xb2\\x9c\\xec\\xa7\\x80 \\xeb\\xa5\\xbc \\xec\\xb0\\xbd\\xec\\xa1\\xb0 \\xed\\x95\\x98 \\xec\\x8b\\x9c \\xec\\x97\\x88 \\xeb\\x8b\\xa4 .'>, <tf.Tensor: shape=(), dtype=string, numpy=b'\\xed\\x83\\x9c\\xec\\xb4\\x88 \\xec\\x97\\x90 \\xed\\x95\\x98\\xeb\\x82\\x98\\xeb\\x8b\\x98 \\xec\\x9d\\xb4 \\xec\\xb2\\x9c\\xec\\xa7\\x80 \\xeb\\xa5\\xbc \\xec\\xb0\\xbd\\xec\\xa1\\xb0 \\xed\\x95\\x98 \\xec\\x8b\\x9c \\xeb\\x8b\\x88\\xeb\\x9d\\xbc'>)\n",
            "(<tf.Tensor: shape=(), dtype=string, numpy=b'\\xeb\\x95\\x85 \\xec\\x9d\\xb4 \\xed\\x98\\xbc\\xeb\\x8f\\x88 \\xed\\x95\\x98 \\xea\\xb3\\xa0 \\xea\\xb3\\xb5\\xed\\x97\\x88 \\xed\\x95\\x98 \\xeb\\xa9\\xb0 , \\xec\\x96\\xb4\\xeb\\x91\\xa0 \\xec\\x9d\\xb4 \\xea\\xb9\\x8a \\xec\\x9d\\x8c \\xec\\x9c\\x84 \\xec\\x97\\x90 \\xec\\x9e\\x88 \\xea\\xb3\\xa0 , \\xed\\x95\\x98 \\xeb\\x82\\x98 \\xeb\\x8b\\x98 \\xec\\x9d\\x98 \\xec\\x98\\x81 \\xec\\x9d\\x80 \\xeb\\xac\\xbc \\xec\\x9c\\x84 \\xec\\x97\\x90 \\xec\\x9b\\x80\\xec\\xa7\\x81\\xec\\x9d\\xb4 \\xea\\xb3\\xa0 \\xea\\xb3\\x84\\xec\\x8b\\x9c \\xec\\x97\\x88 \\xeb\\x8b\\xa4 .'>, <tf.Tensor: shape=(), dtype=string, numpy=b'\\xeb\\x95\\x85 \\xec\\x9d\\xb4 \\xed\\x98\\xbc\\xeb\\x8f\\x88 \\xed\\x95\\x98 \\xea\\xb3\\xa0 \\xea\\xb3\\xb5\\xed\\x97\\x88 \\xed\\x95\\x98 \\xeb\\xa9\\xb0 \\xed\\x9d\\x91 \\xec\\x95\\x94 \\xec\\x9d\\xb4 \\xea\\xb9\\x8a \\xec\\x9d\\x8c \\xec\\x9c\\x84 \\xec\\x97\\x90 \\xec\\x9e\\x88 \\xea\\xb3\\xa0 \\xed\\x95\\x98\\xeb\\x82\\x98\\xeb\\x8b\\x98 \\xec\\x9d\\x98 \\xec\\x98\\x81 \\xec\\x9d\\x80 \\xec\\x88\\x98\\xeb\\xa9\\xb4 \\xec\\x9c\\x84 \\xec\\x97\\x90 \\xec\\x9a\\xb4\\xed\\x96\\x89 \\xed\\x95\\x98 \\xec\\x8b\\x9c \\xeb\\x8b\\x88\\xeb\\x9d\\xbc'>)\n",
            "(<tf.Tensor: shape=(), dtype=string, numpy=b'\\xed\\x95\\x98\\xeb\\x82\\x98\\xeb\\x8b\\x98 \\xec\\x9d\\xb4 \\xeb\\xa7\\x90\\xec\\x94\\x80 \\xed\\x95\\x98 \\xec\\x8b\\x9c \\xea\\xb8\\xb0 \\xeb\\xa5\\xbc \" \\xeb\\xb9\\x9b \\xec\\x9d\\xb4 \\xec\\x83\\x9d\\xea\\xb8\\xb0 \\xec\\x96\\xb4\\xeb\\x9d\\xbc \" \\xed\\x95\\x98 \\xec\\x8b\\x9c \\xeb\\x8b\\x88 , \\xeb\\xb9\\x9b \\xec\\x9d\\xb4 \\xec\\x83\\x9d\\xea\\xb8\\xb0 \\xec\\x97\\x88 \\xeb\\x8b\\xa4 .'>, <tf.Tensor: shape=(), dtype=string, numpy=b'\\xed\\x95\\x98\\xeb\\x82\\x98\\xeb\\x8b\\x98 \\xec\\x9d\\xb4 \\xec\\x9d\\xb4 \\xeb\\xa5\\xb4\\xec\\x8b\\x9c \\xeb\\x90\\x98 \\xeb\\xb9\\x9b \\xec\\x9d\\xb4 \\xec\\x9e\\x88 \\xec\\x9c\\xbc\\xeb\\x9d\\xbc \\xed\\x95\\x98 \\xec\\x8b\\x9c \\xeb\\x8b\\x88 \\xeb\\xb9\\x9b \\xec\\x9d\\xb4 \\xec\\x9e\\x88 \\xec\\x97\\x88 \\xea\\xb3\\xa0'>)\n",
            "(<tf.Tensor: shape=(), dtype=string, numpy=b'\\xea\\xb7\\xb8 \\xeb\\xb9\\x9b \\xec\\x9d\\xb4 \\xed\\x95\\x98\\xeb\\x82\\x98\\xeb\\x8b\\x98 \\xeb\\xb3\\xb4 \\xec\\x8b\\x9c \\xea\\xb8\\xb0 \\xec\\x97\\x90 \\xec\\xa2\\x8b \\xec\\x95\\x98 \\xeb\\x8b\\xa4 . \\xed\\x95\\x98\\xeb\\x82\\x98\\xeb\\x8b\\x98 \\xec\\x9d\\xb4 \\xeb\\xb9\\x9b \\xea\\xb3\\xbc \\xec\\x96\\xb4\\xeb\\x91\\xa0 \\xec\\x9d\\x84 \\xeb\\x82\\x98\\xeb\\x88\\x84 \\xec\\x8b\\x9c \\xec\\x96\\xb4\\xec\\x84\\x9c ,'>, <tf.Tensor: shape=(), dtype=string, numpy=b'\\xeb\\xb9\\x9b \\xec\\x9d\\xb4 \\xed\\x95\\x98\\xeb\\x82\\x98\\xeb\\x8b\\x98 \\xec\\x9d\\xb4 \\xeb\\xb3\\xb4 \\xec\\x8b\\x9c \\xea\\xb8\\xb0 \\xec\\x97\\x90 \\xec\\xa2\\x8b \\xec\\x95\\x98 \\xeb\\x8d\\x94\\xeb\\x9d\\xbc \\xed\\x95\\x98\\xeb\\x82\\x98\\xeb\\x8b\\x98 \\xec\\x9d\\xb4 \\xeb\\xb9\\x9b \\xea\\xb3\\xbc \\xec\\x96\\xb4\\xeb\\x91\\xa0 \\xec\\x9d\\x84 \\xeb\\x82\\x98\\xeb\\x88\\x84 \\xec\\x82\\xac'>)\n",
            "(<tf.Tensor: shape=(), dtype=string, numpy=b'\\xeb\\xb9\\x9b \\xec\\x9d\\x84 \\xeb\\x82\\xae \\xec\\x9d\\xb4 \\xeb\\x9d\\xbc\\xea\\xb3\\xa0 \\xed\\x95\\x98 \\xec\\x8b\\x9c \\xea\\xb3\\xa0 , \\xec\\x96\\xb4\\xeb\\x91\\xa0 \\xec\\x9d\\x84 \\xeb\\xb0\\xa4 \\xec\\x9d\\xb4 \\xeb\\x9d\\xbc\\xea\\xb3\\xa0 \\xed\\x95\\x98 \\xec\\x8b\\x9c \\xec\\x97\\x88 \\xeb\\x8b\\xa4 . \\xec\\xa0\\x80\\xeb\\x85\\x81 \\xec\\x9d\\xb4 \\xeb\\x90\\x98 \\xea\\xb3\\xa0 \\xec\\x95\\x84\\xec\\xb9\\xa8 \\xec\\x9d\\xb4 \\xeb\\x90\\x98 \\xeb\\x8b\\x88 , \\xed\\x95\\x98\\xeb\\xa3\\xa8 \\xea\\xb0\\x80 \\xec\\xa7\\x80\\xeb\\x82\\x98 \\xec\\x95\\x98 \\xeb\\x8b\\xa4 .'>, <tf.Tensor: shape=(), dtype=string, numpy=b'\\xed\\x95\\x98\\xeb\\x82\\x98\\xeb\\x8b\\x98 \\xec\\x9d\\xb4 \\xeb\\xb9\\x9b \\xec\\x9d\\x84 \\xeb\\x82\\xae \\xec\\x9d\\xb4 \\xeb\\x9d\\xbc \\xeb\\xb6\\x80\\xeb\\xa5\\xb4 \\xec\\x8b\\x9c \\xea\\xb3\\xa0 \\xec\\x96\\xb4\\xeb\\x91\\xa0 \\xec\\x9d\\x84 \\xeb\\xb0\\xa4 \\xec\\x9d\\xb4 \\xeb\\x9d\\xbc \\xeb\\xb6\\x80\\xeb\\xa5\\xb4 \\xec\\x8b\\x9c \\xeb\\x8b\\x88\\xeb\\x9d\\xbc \\xec\\xa0\\x80\\xeb\\x85\\x81 \\xec\\x9d\\xb4 \\xeb\\x90\\x98 \\xea\\xb3\\xa0 \\xec\\x95\\x84\\xec\\xb9\\xa8 \\xec\\x9d\\xb4 \\xeb\\x90\\x98 \\xeb\\x8b\\x88 \\xec\\x9d\\xb4 \\xeb\\x8a\\x94 \\xec\\xb2\\xab\\xec\\xa7\\xb8 \\xeb\\x82\\xa0 \\xec\\x9d\\xb4 \\xeb\\x8b\\x88\\xeb\\x9d\\xbc'>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y66ttD3_c8Dc"
      },
      "source": [
        "train_dataset = lines_dataset.map(tf_encode)\n",
        "train_dataset = train_dataset.cache()\n",
        "train_dataset = train_dataset.shuffle(BUFFER_SIZE).padded_batch(BATCH_SIZE, padded_shapes=([-1], [-1]))\n",
        "train_dataset = train_dataset.prefetch(tf.data.experimental.AUTOTUNE)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tVNjJiOIdoMH",
        "outputId": "05c7cc04-8cb1-43d0-b77a-e89957fd2012",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        }
      },
      "source": [
        "en_batch, kr_batch = next(iter(train_dataset))\n",
        "print(en_batch, kr_batch)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor(\n",
            "[[9504    8   11 ...    0    0    0]\n",
            " [9504  369    1 ...    0    0    0]\n",
            " [9504  188   21 ...    0    0    0]\n",
            " ...\n",
            " [9504    8   11 ...    0    0    0]\n",
            " [9504   54    5 ...    0    0    0]\n",
            " [9504    8 1568 ...    0    0    0]], shape=(64, 88), dtype=int64) tf.Tensor(\n",
            "[[10609     5     3 ...     0     0     0]\n",
            " [10609   344     1 ...     0     0     0]\n",
            " [10609   144    29 ...     0     0     0]\n",
            " ...\n",
            " [10609     5     6 ...     0     0     0]\n",
            " [10609    51   215 ...     0     0     0]\n",
            " [10609     5    10 ...     0     0     0]], shape=(64, 82), dtype=int64)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TmWsEIgidv5l"
      },
      "source": [
        "Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-j6Z5TzVdpeO"
      },
      "source": [
        "def get_angles(pos, i, d_model):\n",
        "    angle_rates = 1 / np.power(10000, (2 * (i//2)) / np.float32(d_model))\n",
        "    return pos * angle_rates\n",
        "\n",
        "def positional_encoding(position, d_model):\n",
        "    angle_rads = get_angles(np.arange(position)[:, np.newaxis], np.arange(d_model)[np.newaxis, :], d_model)\n",
        "    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
        "    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
        "    \n",
        "    pos_encoding = angle_rads[np.newaxis, ...]\n",
        "    \n",
        "    return tf.cast(pos_encoding, dtype=tf.float32)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pcur4wkDdzJO"
      },
      "source": [
        "def create_padding_mask(seq):\n",
        "    seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
        "    return seq[:, tf.newaxis, tf.newaxis, :]"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kHXhJuuad0nH"
      },
      "source": [
        "def create_look_ahead_mask(size):\n",
        "    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n",
        "    return mask"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z3tLE6Iud1p-"
      },
      "source": [
        "def scaled_dot_product_attention(q, k, v, mask):\n",
        "    matmul_qk = tf.matmul(q, k, transpose_b=True)\n",
        "    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
        "    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
        "\n",
        "    if mask is not None:\n",
        "        scaled_attention_logits += (mask * -1e9)  \n",
        "\n",
        "    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)\n",
        "\n",
        "    output = tf.matmul(attention_weights, v)\n",
        "\n",
        "    return output, attention_weights"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ceWy4Ygzd21W"
      },
      "source": [
        "class MultiHeadAttention(tf.keras.layers.Layer):\n",
        "    def __init__(self, d_model, num_heads):\n",
        "        super(MultiHeadAttention, self).__init__()\n",
        "        self.num_heads = num_heads\n",
        "        self.d_model = d_model\n",
        "    \n",
        "        assert d_model % self.num_heads == 0\n",
        "    \n",
        "        self.depth = d_model // self.num_heads\n",
        "    \n",
        "        self.wq = tf.keras.layers.Dense(d_model)\n",
        "        self.wk = tf.keras.layers.Dense(d_model)\n",
        "        self.wv = tf.keras.layers.Dense(d_model)\n",
        "    \n",
        "        self.dense = tf.keras.layers.Dense(d_model)\n",
        "        \n",
        "    def split_heads(self, x, batch_size):\n",
        "        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
        "        return tf.transpose(x, perm=[0, 2, 1, 3])\n",
        "    \n",
        "    def call(self, v, k, q, mask):\n",
        "        batch_size = tf.shape(q)[0]\n",
        "    \n",
        "        q = self.wq(q)\n",
        "        k = self.wk(k)\n",
        "        v = self.wv(v)\n",
        "    \n",
        "        q = self.split_heads(q, batch_size)\n",
        "        k = self.split_heads(k, batch_size)\n",
        "        v = self.split_heads(v, batch_size)\n",
        "    \n",
        "        scaled_attention, attention_weights = scaled_dot_product_attention(q, k, v, mask)\n",
        "    \n",
        "        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])\n",
        "\n",
        "        concat_attention = tf.reshape(scaled_attention, \n",
        "                                  (batch_size, -1, self.d_model))\n",
        "\n",
        "        output = self.dense(concat_attention)\n",
        "        \n",
        "        return output, attention_weights"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0MGasaKdd4yW"
      },
      "source": [
        "def point_wise_feed_forward_network(d_model, dff):\n",
        "    return tf.keras.Sequential([tf.keras.layers.Dense(dff, activation='relu'), tf.keras.layers.Dense(d_model)])"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bxoC-2yhd6Le"
      },
      "source": [
        "class EncoderLayer(tf.keras.layers.Layer):\n",
        "    def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "\n",
        "        self.mha = MultiHeadAttention(d_model, num_heads)\n",
        "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        "\n",
        "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    \n",
        "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "    \n",
        "    def call(self, x, training, mask):\n",
        "\n",
        "        attn_output, _ = self.mha(x, x, x, mask)\n",
        "        attn_output = self.dropout1(attn_output, training=training)\n",
        "        out1 = self.layernorm1(x + attn_output)\n",
        "    \n",
        "        ffn_output = self.ffn(out1)\n",
        "        ffn_output = self.dropout2(ffn_output, training=training)\n",
        "        out2 = self.layernorm2(out1 + ffn_output)\n",
        "    \n",
        "        return out2"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_tUClcdpd8yH"
      },
      "source": [
        "class DecoderLayer(tf.keras.layers.Layer):\n",
        "    def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
        "        super(DecoderLayer, self).__init__()\n",
        "\n",
        "        self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
        "        self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
        "\n",
        "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
        "        \n",
        "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "        self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
        "    \n",
        "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
        "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
        "        self.dropout3 = tf.keras.layers.Dropout(rate)\n",
        "    \n",
        "    \n",
        "    def call(self, x, enc_output, training, look_ahead_mask, padding_mask):\n",
        "\n",
        "        attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)\n",
        "        attn1 = self.dropout1(attn1, training=training)\n",
        "        out1 = self.layernorm1(attn1 + x)\n",
        "    \n",
        "        attn2, attn_weights_block2 = self.mha2(enc_output, enc_output, out1, padding_mask)\n",
        "        attn2 = self.dropout2(attn2, training=training)\n",
        "        out2 = self.layernorm2(attn2 + out1)\n",
        "    \n",
        "        ffn_output = self.ffn(out2) \n",
        "        ffn_output = self.dropout3(ffn_output, training=training)\n",
        "        out3 = self.layernorm3(ffn_output + out2)\n",
        "    \n",
        "        return out3, attn_weights_block1, attn_weights_block2"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wBaJA0ixd-cW"
      },
      "source": [
        "class Encoder(tf.keras.layers.Layer):\n",
        "    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, maximum_position_encoding, rate=0.1):\n",
        "        super(Encoder, self).__init__()\n",
        "\n",
        "        self.d_model = d_model\n",
        "        self.num_layers = num_layers\n",
        "    \n",
        "        self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n",
        "        self.pos_encoding = positional_encoding(maximum_position_encoding, self.d_model)\n",
        "\n",
        "        self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)]\n",
        "        \n",
        "        self.dropout = tf.keras.layers.Dropout(rate)\n",
        "        \n",
        "    def call(self, x, training, mask):\n",
        "\n",
        "        seq_len = tf.shape(x)[1]\n",
        "    \n",
        "        x = self.embedding(x)\n",
        "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "        x += self.pos_encoding[:, :seq_len, :]\n",
        "\n",
        "        x = self.dropout(x, training=training)\n",
        "    \n",
        "        for i in range(self.num_layers):\n",
        "            x = self.enc_layers[i](x, training, mask)\n",
        "    \n",
        "        return x"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UUv0SRRud_3_"
      },
      "source": [
        "class Decoder(tf.keras.layers.Layer):\n",
        "    def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size, maximum_position_encoding, rate=0.1):\n",
        "        super(Decoder, self).__init__()\n",
        "\n",
        "        self.d_model = d_model\n",
        "        self.num_layers = num_layers\n",
        "    \n",
        "        self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n",
        "        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n",
        "    \n",
        "        self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)]\n",
        "        self.dropout = tf.keras.layers.Dropout(rate)\n",
        "    \n",
        "    def call(self, x, enc_output, training, look_ahead_mask, padding_mask):\n",
        "\n",
        "        seq_len = tf.shape(x)[1]\n",
        "        attention_weights = {}\n",
        "    \n",
        "        x = self.embedding(x)\n",
        "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
        "        x += self.pos_encoding[:, :seq_len, :]\n",
        "    \n",
        "        x = self.dropout(x, training=training)\n",
        "\n",
        "        for i in range(self.num_layers):\n",
        "            x, block1, block2 = self.dec_layers[i](x, enc_output, training,look_ahead_mask, padding_mask)\n",
        "            attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n",
        "            attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n",
        "    \n",
        "        return x, attention_weights"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yg6R_SWEeBN_"
      },
      "source": [
        "class Transformer(tf.keras.Model):\n",
        "    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, pe_input, pe_target, rate=0.1):\n",
        "        super(Transformer, self).__init__()\n",
        "        self.encoder = Encoder(num_layers, d_model, num_heads, dff, input_vocab_size, pe_input, rate)\n",
        "        self.decoder = Decoder(num_layers, d_model, num_heads, dff, target_vocab_size, pe_target, rate)\n",
        "        self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n",
        "    \n",
        "    def call(self, inp, tar, training, enc_padding_mask, look_ahead_mask, dec_padding_mask):\n",
        "\n",
        "        enc_output = self.encoder(inp, training, enc_padding_mask)\n",
        "        dec_output, attention_weights = self.decoder(tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n",
        "    \n",
        "        final_output = self.final_layer(dec_output)\n",
        "    \n",
        "        return final_output, attention_weights"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X_KpFB0ueCqe"
      },
      "source": [
        "num_layers = 4\n",
        "d_model = 128\n",
        "dff = 512\n",
        "num_heads = 8\n",
        "\n",
        "input_vocab_size = tokenizer_source.vocab_size + 2\n",
        "target_vocab_size = tokenizer_target.vocab_size + 2\n",
        "dropout_rate = 0.1"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7V51s55feElG"
      },
      "source": [
        "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
        "    def __init__(self, d_model, warmup_steps=4000):\n",
        "        super(CustomSchedule, self).__init__()\n",
        "    \n",
        "        self.d_model = d_model\n",
        "        self.d_model = tf.cast(self.d_model, tf.float32)\n",
        "\n",
        "        self.warmup_steps = warmup_steps\n",
        "    \n",
        "    def __call__(self, step):\n",
        "        arg1 = tf.math.rsqrt(step)\n",
        "        arg2 = step * (self.warmup_steps ** -1.5)\n",
        "    \n",
        "        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Axs_uTreGdH"
      },
      "source": [
        "learning_rate = CustomSchedule(d_model)\n",
        "# learning_rate = 1e-3\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, \n",
        "                                     epsilon=1e-9)\n",
        "# optimizer = tf.keras.optimizers.Adam(learning_rate)"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_l9D48zCfOu-"
      },
      "source": [
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
        "    from_logits=True, reduction='none')"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1opuZyLMeH4e"
      },
      "source": [
        "def loss_function(real, pred):\n",
        "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "    loss_ = loss_object(real, pred)\n",
        "\n",
        "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "    loss_ *= mask\n",
        "    \n",
        "    return tf.reduce_mean(loss_)"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pEOc57HfewEw"
      },
      "source": [
        "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
        "train_accuracy = tf.keras.metrics.SparseCategoricalAccuracy(\n",
        "    name='train_accuracy')"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jKvWfBuWexI1"
      },
      "source": [
        "transformer = Transformer(num_layers, d_model, num_heads, dff,\n",
        "                          input_vocab_size, target_vocab_size, \n",
        "                          pe_input=input_vocab_size, \n",
        "                          pe_target=target_vocab_size,\n",
        "                          rate=dropout_rate)"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "era4UN63eylV"
      },
      "source": [
        "def create_masks(inp, tar):\n",
        "    enc_padding_mask = create_padding_mask(inp)\n",
        "    dec_padding_mask = create_padding_mask(inp)\n",
        "\n",
        "    look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n",
        "    dec_target_padding_mask = create_padding_mask(tar)\n",
        "    combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n",
        "    \n",
        "    return enc_padding_mask, combined_mask, dec_padding_mask"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FcAKFq_WezxG"
      },
      "source": [
        "checkpoint_path = '/content/bible_translator'\n",
        "\n",
        "ckpt = tf.train.Checkpoint(transformer=transformer,\n",
        "                           optimizer=optimizer)\n",
        "\n",
        "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n",
        "\n",
        "if ckpt_manager.latest_checkpoint:\n",
        "    ckpt.restore(ckpt_manager.latest_checkpoint)\n",
        "    print ('Latest checkpoint is restored')"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "omBzx7Sie6s-"
      },
      "source": [
        "EPOCHS = 5"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "viaItmdoe77m"
      },
      "source": [
        "train_step_signature = [\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "    tf.TensorSpec(shape=(None, None), dtype=tf.int64),\n",
        "]\n",
        "\n",
        "@tf.function(input_signature=train_step_signature)\n",
        "def train_step(inp, tar):\n",
        "    tar_inp = tar[:, :-1]\n",
        "    tar_real = tar[:, 1:]\n",
        "    \n",
        "    enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n",
        "    \n",
        "    with tf.GradientTape() as tape:\n",
        "        predictions, _ = transformer(inp, tar_inp, True, enc_padding_mask, combined_mask, dec_padding_mask)\n",
        "        loss = loss_function(tar_real, predictions)\n",
        "\n",
        "    gradients = tape.gradient(loss, transformer.trainable_variables)    \n",
        "    optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
        "    train_loss(loss)\n",
        "    train_accuracy(tar_real, predictions)"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dZ0esuYue-fu",
        "outputId": "a54e933f-4290-4242-9628-ade2ff9b61fe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "for epoch in range(EPOCHS):\n",
        "    start = time.time()\n",
        "    train_loss.reset_states()\n",
        "    train_accuracy.reset_states()\n",
        "    \n",
        "    for (batch, (inp, tar)) in enumerate(train_dataset):\n",
        "        train_step(inp, tar)\n",
        "    \n",
        "        if batch % 50 == 0:\n",
        "            print ('Epoch {} Batch {} Loss {:.4f} Accuracy {:.4f}'.format(epoch + 1, batch, train_loss.result(), train_accuracy.result()))\n",
        "            \n",
        "    if (epoch + 1) % 5 == 0:\n",
        "        ckpt_save_path = ckpt_manager.save()\n",
        "        print ('Saving checkpoint for epoch {} at {}'.format(epoch+1, ckpt_save_path))\n",
        "    \n",
        "    print ('Epoch {} Loss {:.4f} Accuracy {:.4f}'.format(epoch + 1, train_loss.result(), train_accuracy.result()))\n",
        "\n",
        "    print ('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1 Batch 0 Loss 4.1601 Accuracy 0.0000\n",
            "Epoch 1 Batch 50 Loss 4.2216 Accuracy 0.0024\n",
            "Epoch 1 Batch 100 Loss 4.1677 Accuracy 0.0084\n",
            "Epoch 1 Batch 150 Loss 4.1057 Accuracy 0.0117\n",
            "Epoch 1 Batch 200 Loss 4.0129 Accuracy 0.0139\n",
            "Epoch 1 Batch 250 Loss 3.9110 Accuracy 0.0154\n",
            "Epoch 1 Batch 300 Loss 3.7999 Accuracy 0.0166\n",
            "Epoch 1 Batch 350 Loss 3.6821 Accuracy 0.0177\n",
            "Epoch 1 Batch 400 Loss 3.5768 Accuracy 0.0193\n",
            "Epoch 1 Batch 450 Loss 3.4733 Accuracy 0.0212\n",
            "Epoch 1 Loss 3.4148 Accuracy 0.0229\n",
            "Time taken for 1 epoch: 81.58805179595947 secs\n",
            "\n",
            "Epoch 2 Batch 0 Loss 2.4411 Accuracy 0.0487\n",
            "Epoch 2 Batch 50 Loss 2.5418 Accuracy 0.0520\n",
            "Epoch 2 Batch 100 Loss 2.5123 Accuracy 0.0586\n",
            "Epoch 2 Batch 150 Loss 2.4649 Accuracy 0.0653\n",
            "Epoch 2 Batch 200 Loss 2.3952 Accuracy 0.0704\n",
            "Epoch 2 Batch 250 Loss 2.3559 Accuracy 0.0757\n",
            "Epoch 2 Batch 300 Loss 2.3111 Accuracy 0.0797\n",
            "Epoch 2 Batch 350 Loss 2.2733 Accuracy 0.0832\n",
            "Epoch 2 Batch 400 Loss 2.2336 Accuracy 0.0859\n",
            "Epoch 2 Batch 450 Loss 2.2118 Accuracy 0.0892\n",
            "Epoch 2 Loss 2.1915 Accuracy 0.0910\n",
            "Time taken for 1 epoch: 54.92818880081177 secs\n",
            "\n",
            "Epoch 3 Batch 0 Loss 1.8731 Accuracy 0.1172\n",
            "Epoch 3 Batch 50 Loss 1.8990 Accuracy 0.1175\n",
            "Epoch 3 Batch 100 Loss 1.8815 Accuracy 0.1178\n",
            "Epoch 3 Batch 150 Loss 1.8826 Accuracy 0.1198\n",
            "Epoch 3 Batch 200 Loss 1.8803 Accuracy 0.1212\n",
            "Epoch 3 Batch 250 Loss 1.8648 Accuracy 0.1217\n",
            "Epoch 3 Batch 300 Loss 1.8553 Accuracy 0.1225\n",
            "Epoch 3 Batch 350 Loss 1.8491 Accuracy 0.1238\n",
            "Epoch 3 Batch 400 Loss 1.8369 Accuracy 0.1243\n",
            "Epoch 3 Batch 450 Loss 1.8301 Accuracy 0.1254\n",
            "Epoch 3 Loss 1.8250 Accuracy 0.1260\n",
            "Time taken for 1 epoch: 54.83979797363281 secs\n",
            "\n",
            "Epoch 4 Batch 0 Loss 1.6294 Accuracy 0.1528\n",
            "Epoch 4 Batch 50 Loss 1.7333 Accuracy 0.1403\n",
            "Epoch 4 Batch 100 Loss 1.7159 Accuracy 0.1396\n",
            "Epoch 4 Batch 150 Loss 1.6976 Accuracy 0.1384\n",
            "Epoch 4 Batch 200 Loss 1.6844 Accuracy 0.1380\n",
            "Epoch 4 Batch 250 Loss 1.6883 Accuracy 0.1392\n",
            "Epoch 4 Batch 300 Loss 1.6833 Accuracy 0.1397\n",
            "Epoch 4 Batch 350 Loss 1.6765 Accuracy 0.1399\n",
            "Epoch 4 Batch 400 Loss 1.6799 Accuracy 0.1411\n",
            "Epoch 4 Batch 450 Loss 1.6744 Accuracy 0.1417\n",
            "Epoch 4 Loss 1.6719 Accuracy 0.1421\n",
            "Time taken for 1 epoch: 54.72224974632263 secs\n",
            "\n",
            "Epoch 5 Batch 0 Loss 1.4016 Accuracy 0.1404\n",
            "Epoch 5 Batch 50 Loss 1.5625 Accuracy 0.1504\n",
            "Epoch 5 Batch 100 Loss 1.5741 Accuracy 0.1514\n",
            "Epoch 5 Batch 150 Loss 1.5742 Accuracy 0.1516\n",
            "Epoch 5 Batch 200 Loss 1.5645 Accuracy 0.1518\n",
            "Epoch 5 Batch 250 Loss 1.5601 Accuracy 0.1513\n",
            "Epoch 5 Batch 300 Loss 1.5606 Accuracy 0.1520\n",
            "Epoch 5 Batch 350 Loss 1.5590 Accuracy 0.1522\n",
            "Epoch 5 Batch 400 Loss 1.5577 Accuracy 0.1529\n",
            "Epoch 5 Batch 450 Loss 1.5550 Accuracy 0.1537\n",
            "Saving checkpoint for epoch 5 at /content/bible_translator/ckpt-1\n",
            "Epoch 5 Loss 1.5465 Accuracy 0.1537\n",
            "Time taken for 1 epoch: 55.56697058677673 secs\n",
            "\n",
            "Epoch 6 Batch 0 Loss 1.1518 Accuracy 0.1390\n",
            "Epoch 6 Batch 50 Loss 1.4683 Accuracy 0.1677\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-40-17ed462b1cbf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0minp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m         \u001b[0mtrain_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtar\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mbatch\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m50\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    778\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    779\u001b[0m         \u001b[0mcompiler\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"nonXla\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 780\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    781\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    782\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    805\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    806\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 807\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    808\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    809\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   2827\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2828\u001b[0m       \u001b[0mgraph_function\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_define_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2829\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mgraph_function\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_filtered_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=protected-access\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2830\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2831\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_filtered_call\u001b[0;34m(self, args, kwargs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1846\u001b[0m                            resource_variable_ops.BaseResourceVariable))],\n\u001b[1;32m   1847\u001b[0m         \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1848\u001b[0;31m         cancellation_manager=cancellation_manager)\n\u001b[0m\u001b[1;32m   1849\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1850\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_flat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcaptured_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcancellation_manager\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1922\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1923\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1924\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1925\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1926\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    548\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    551\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H9rHpuJKfDVu"
      },
      "source": [
        "def evaluate(inp_sentence):\n",
        "    start_token = [tokenizer_source.vocab_size]\n",
        "    end_token = [tokenizer_source.vocab_size + 1]\n",
        "    \n",
        "    inp_sentence = start_token + tokenizer_source.encode(inp_sentence) + end_token\n",
        "    encoder_input = tf.expand_dims(inp_sentence, 0)\n",
        "\n",
        "    decoder_input = [tokenizer_target.vocab_size]\n",
        "    output = tf.expand_dims(decoder_input, 0)\n",
        "    \n",
        "    for i in range(MAX_LENGTH):\n",
        "        enc_padding_mask, combined_mask, dec_padding_mask = create_masks(encoder_input, output)\n",
        "        predictions, attention_weights = transformer(encoder_input, output, False, enc_padding_mask, combined_mask, dec_padding_mask)\n",
        "    \n",
        "        predictions = predictions[: ,-1:, :]  # (batch_size, 1, vocab_size)\n",
        "\n",
        "        predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
        "        \n",
        "        if predicted_id >= tokenizer_target.vocab_size+1:\n",
        "            return tf.squeeze(output, axis=0), attention_weights\n",
        "        \n",
        "        output = tf.concat([output, predicted_id], axis=-1)\n",
        "\n",
        "    return tf.squeeze(output, axis=0), attention_weights"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RKIq7gk1gfwb"
      },
      "source": [
        "def prediction(sentence):\n",
        "    result, attention_weights = evaluate(sentence)\n",
        "    predicted_sentence = tokenizer_target.decode([i for i in result \n",
        "                                            if i < tokenizer_target.vocab_size])  \n",
        "    print('Bible: {}'.format(predicted_sentence))"
      ],
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LvDN4aq2YPva",
        "outputId": "a1e7aa3b-e9ef-4f10-8a57-f89ab5ba64ac",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "prediction('집 에 가 고 싶 다 .')"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Bible: 네 가 집 에 들어가 아서 죽 은 것 을 보 고 마시 니\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9C-13Of_nwGa"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}