training_model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "training_model.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "JPnlWgkxgyxd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "520cc6a9-b34c-43d7-e92b-8971f7ced518"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import keras\n",
        "import re\n",
        "import nltk\n",
        "from nltk.corpus import stopwords\n",
        "import string\n",
        "import json\n",
        "from time import time\n",
        "import pickle\n",
        "from keras.applications.vgg16 import VGG16\n",
        "from keras.applications.resnet50 import ResNet50, preprocess_input, decode_predictions\n",
        "from keras.preprocessing import image\n",
        "from keras.models import Model, load_model\n",
        "from keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import to_categorical\n",
        "from keras.layers import Input, Dense, Dropout, Embedding, LSTM\n",
        "from keras.layers.merge import add\n",
        "import collections"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2txQyrGSjcnP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "aeff23a2-de02-4d0b-b6ff-ab10e09d014c"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive',force_remount=True)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vtCHjtdUg25S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "index_to_word = None\n",
        "with open(\"index_to_word.pkl\",'rb') as f:\n",
        "  index_to_word = pickle.load(f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7d1xgtCliEn4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "word_to_index = None\n",
        "with open(\"word_to_index.pkl\",'rb') as f:\n",
        "  word_to_index = pickle.load(f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LlNswKq3idiC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embedding_matrix = None\n",
        "with open(\"embedding_matrix.pkl\",'rb') as f:\n",
        "  embedding_matrix = pickle.load(f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k150WUmUimwf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "encoding_train = None\n",
        "with open(\"/content/gdrive/My Drive/Data_Science/Image_captioning_bot/30k/encoding_train_features.pkl\",'rb') as f:\n",
        "  encoding_train = pickle.load(f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d693NG0uj33a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_descriptions = None\n",
        "with open(\"train_descriptions.pkl\",'rb') as f:\n",
        "  train_descriptions = pickle.load(f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QX4gLoM-kfoe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "c637c1f8-b321-4d86-f98b-9266f7f14ab1"
      },
      "source": [
        "max_len = 0\n",
        "for key in train_descriptions.keys():\n",
        "    for cap in train_descriptions[key]:\n",
        "        max_len = max(max_len,len(cap.split()))\n",
        "print(max_len)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "74\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-uxoLqEyk8zd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "42dea09b-d64b-4920-fcfc-b46e5f647706"
      },
      "source": [
        "vocab_size = len(word_to_index)+1\n",
        "print(vocab_size)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "5121\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q7JhQJCYkP3l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def data_generator(train_descriptions,encoding_train,word_to_index,max_len,batch_size):\n",
        "    X1,X2,Y = [],[],[]\n",
        "    \n",
        "    n=0\n",
        "    while True:\n",
        "        for key, desc_list in train_descriptions.items():\n",
        "            n+=1\n",
        "            \n",
        "            photo = encoding_train[key]\n",
        "            for desc in desc_list:\n",
        "                seq = [word_to_index[word] for word in desc.split() if word in word_to_index]\n",
        "                for i in range(1,len(seq)):\n",
        "                    xi = seq[0:i]\n",
        "                    yi = seq[i]\n",
        "                    \n",
        "                    xi = pad_sequences([xi],maxlen=max_len,value=0,padding='post')[0]\n",
        "                    yi = to_categorical([yi],num_classes=vocab_size)[0]\n",
        "                    \n",
        "                    X1.append(photo)\n",
        "                    X2.append(xi)\n",
        "                    Y.append(yi)\n",
        "                    \n",
        "                if n==batch_size:\n",
        "                    yield [[np.array(X1),np.array(X2)],np.array(Y)]\n",
        "                    \n",
        "                    X1,X2,Y = [],[],[]\n",
        "                    n=0"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P2ywUKIqks0n",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "input_img_features = Input(shape=(2048,))\n",
        "inp_img1 = Dropout(0.1)(input_img_features)\n",
        "inp_img2 = Dense(256,activation='relu')(inp_img1)\n",
        "\n",
        "input_captions = Input(shape=(max_len,))\n",
        "inp_cap1 = Embedding(input_dim=vocab_size,output_dim=50,mask_zero=True)(input_captions)\n",
        "inp_cap2 = Dropout(0.1)(inp_cap1)\n",
        "inp_cap3 = LSTM(256)(inp_cap2)\n",
        "\n",
        "decoder1 = add([inp_img2,inp_cap3])\n",
        "decoder2 = Dense(256,activation='relu')(decoder1)\n",
        "outputs = Dense(vocab_size,activation='softmax')(decoder2)\n",
        "\n",
        "model = Model(inputs=[input_img_features,input_captions],outputs=outputs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bApwptP8kcAl",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 510
        },
        "outputId": "dff385d5-33ae-43a3-d152-a0629257ad5a"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_1\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_2 (InputLayer)            (None, 74)           0                                            \n",
            "__________________________________________________________________________________________________\n",
            "input_1 (InputLayer)            (None, 2048)         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "embedding_1 (Embedding)         (None, 74, 50)       256050      input_2[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dropout_1 (Dropout)             (None, 2048)         0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dropout_2 (Dropout)             (None, 74, 50)       0           embedding_1[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 256)          524544      dropout_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "lstm_1 (LSTM)                   (None, 256)          314368      dropout_2[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "add_1 (Add)                     (None, 256)          0           dense_1[0][0]                    \n",
            "                                                                 lstm_1[0][0]                     \n",
            "__________________________________________________________________________________________________\n",
            "dense_2 (Dense)                 (None, 256)          65792       add_1[0][0]                      \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 5121)         1316097     dense_2[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 2,476,851\n",
            "Trainable params: 2,476,851\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5wjMo6BnlFps",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.layers[2].set_weights([embedding_matrix])\n",
        "model.layers[2].trainable = False"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wh2-Fdu7lQl3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "epochs=40\n",
        "batch_size = 64\n",
        "steps = len(train_descriptions)//batch_size"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lsa0DHB6mwag",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.compile(loss=\"categorical_crossentropy\",optimizer=\"adam\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dxslj43poPvk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir model_weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4-p6fwSemyq6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "a89ffb3b-3ce4-4e66-cb14-0f7a6dc0fb85"
      },
      "source": [
        "for i in range(epochs):\n",
        "    generator = data_generator(train_descriptions,encoding_train,word_to_index,max_len,batch_size)\n",
        "    model.fit_generator(generator,epochs=1,steps_per_epoch=steps,verbose=1)\n",
        "    model.save('model_weights/model_'+str(i)+'.h5')"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/1\n",
            "496/496 [==============================] - 207s 418ms/step - loss: 4.7486\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 414ms/step - loss: 3.9258\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 413ms/step - loss: 3.6761\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 207s 417ms/step - loss: 3.5315\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 207s 417ms/step - loss: 3.4315\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 412ms/step - loss: 3.3573\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 410ms/step - loss: 3.2973\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 412ms/step - loss: 3.2483\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 411ms/step - loss: 3.2081\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 412ms/step - loss: 3.1731\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 413ms/step - loss: 3.1428\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 203s 410ms/step - loss: 3.1164\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 414ms/step - loss: 3.0922\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 413ms/step - loss: 3.0718\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 413ms/step - loss: 3.0534\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 411ms/step - loss: 3.0369\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 412ms/step - loss: 3.0213\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 413ms/step - loss: 3.0080\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 410ms/step - loss: 2.9952\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 413ms/step - loss: 2.9840\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 411ms/step - loss: 2.9734\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 411ms/step - loss: 2.9632\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 413ms/step - loss: 2.9524\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 411ms/step - loss: 2.9443\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 203s 409ms/step - loss: 2.9352\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 413ms/step - loss: 2.9271\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 203s 410ms/step - loss: 2.9201\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 412ms/step - loss: 2.9134\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 412ms/step - loss: 2.9074\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 412ms/step - loss: 2.9018\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 203s 410ms/step - loss: 2.8963\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 413ms/step - loss: 2.8923\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 411ms/step - loss: 2.8879\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 410ms/step - loss: 2.8834\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 205s 413ms/step - loss: 2.8784\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 203s 410ms/step - loss: 2.8733\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 411ms/step - loss: 2.8685\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 412ms/step - loss: 2.8640\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 204s 411ms/step - loss: 2.8595\n",
            "Epoch 1/1\n",
            "496/496 [==============================] - 203s 410ms/step - loss: 2.8548\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FaX_YvQ6m_6O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}