13_lstm_pntk.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1410,
     "status": "ok",
     "timestamp": 1586249684569,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "9nn-YIyzJ09Y",
    "outputId": "0f388e74-3a52-4f18-f513-529ab2390ea0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rhUo7OzGQl4Z"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from math import sqrt\n",
    "\n",
    "from tensorflow.keras.layers import Input, Dense, Embedding, LSTM, Activation, GlobalMaxPooling1D\n",
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.model_selection import StratifiedShuffleSplit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2pNg4ay8JZmg"
   },
   "outputs": [],
   "source": [
    "#Run Local\n",
    "#df = pd.read_csv(\"csv/book_info_complete.csv\")\n",
    "#Run on COLAB\n",
    "df = pd.read_csv(\"/content/drive/My Drive/final_project/book_info_complete.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 301
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3342,
     "status": "ok",
     "timestamp": 1586249686536,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "5hNMh6xrJZmm",
    "outputId": "8f982c30-bd43-484a-b9f1-6ee8d843c831"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5816 entries, 0 to 5815\n",
      "Data columns (total 9 columns):\n",
      " #   Column            Non-Null Count  Dtype \n",
      "---  ------            --------------  ----- \n",
      " 0   title             5816 non-null   object\n",
      " 1   description       5816 non-null   object\n",
      " 2   plot              5816 non-null   object\n",
      " 3   csm_review        5816 non-null   object\n",
      " 4   need_to_know      5816 non-null   object\n",
      " 5   csm_rating        5816 non-null   int64 \n",
      " 6   Genre             5816 non-null   object\n",
      " 7   Book type         5816 non-null   object\n",
      " 8   Publication date  5816 non-null   int64 \n",
      "dtypes: int64(2), object(7)\n",
      "memory usage: 409.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df = df.dropna(axis=1)\n",
    "df[\"Publication date\"] = df[\"Publication date\"].str[-4:].astype(int)\n",
    "df = df.drop(\"Last updated\", axis=1)\n",
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "a1RnxUyXJZmx"
   },
   "source": [
    "## Create the splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "M0_-59xOJZmy"
   },
   "outputs": [],
   "source": [
    "def splitter(df):\n",
    "    split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=999)\n",
    "    for train_index, test_index in split.split(df, df['csm_rating']):\n",
    "        train_data= df.loc[train_index]\n",
    "        test_data = df.loc[test_index]\n",
    "    \n",
    "    \n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3wkFIijOJZm4"
   },
   "outputs": [],
   "source": [
    "train_data, test_data = splitter(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YjI0Lfc8JZm9"
   },
   "outputs": [],
   "source": [
    "x_tr, y_tr = train_data['need_to_know'].values, train_data[\"csm_rating\"].values\n",
    "x_val, y_val = test_data[\"need_to_know\"].values, test_data[\"csm_rating\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3309,
     "status": "ok",
     "timestamp": 1586249686539,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "64omNypEJZnC",
    "outputId": "1f364b96-e7ee-4c10-dbe2-7e90a4cb8ff3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4652,) (4652,)\n"
     ]
    }
   ],
   "source": [
    "print(x_tr.shape, y_tr.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3297,
     "status": "ok",
     "timestamp": 1586249686540,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "6SCDNHa2JZnJ",
    "outputId": "c9f2275a-c31e-4f7a-ed90-adb35526736f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1164,) (1164,)\n"
     ]
    }
   ],
   "source": [
    "print(x_val.shape, y_val.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Z1CH5m22JZnO"
   },
   "source": [
    "## Prepare the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4025,
     "status": "ok",
     "timestamp": 1586249687280,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "liitdrp9JZnP",
    "outputId": "cb0b7e02-5eb7-4762-db97-48013bea2848"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "332\n",
      "10\n"
     ]
    }
   ],
   "source": [
    "#Tokenize the sentences\n",
    "tokenizer = Tokenizer()\n",
    "\n",
    "#preparing vocabulary\n",
    "tokenizer.fit_on_texts(list(x_tr))\n",
    "\n",
    "#converting text into integer sequences\n",
    "x_tr_seq  = tokenizer.texts_to_sequences(x_tr) \n",
    "x_val_seq = tokenizer.texts_to_sequences(x_val)\n",
    "\n",
    "print(len(max(x_tr_seq, key=len)))\n",
    "max_length = len(max(x_tr_seq, key=len))\n",
    "print(len(min(x_tr_seq, key=len)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4018,
     "status": "ok",
     "timestamp": 1586249687283,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "nGP4OL9XJZnT",
    "outputId": "77890032-4700-46a2-c151-8285c1876af8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "332\n",
      "332\n"
     ]
    }
   ],
   "source": [
    "#padding to prepare sequences of same length\n",
    "x_tr_seq  = pad_sequences(x_tr_seq, maxlen=max_length)\n",
    "x_val_seq = pad_sequences(x_val_seq, maxlen=max_length)\n",
    "print(len(max(x_tr_seq, key=len)))\n",
    "print(len(min(x_tr_seq, key=len)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4008,
     "status": "ok",
     "timestamp": 1586249687284,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "NuoM9u8HJZnZ",
    "outputId": "2f1b9428-6519-47ce-b026-752c9bcce68a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24351\n"
     ]
    }
   ],
   "source": [
    "size_of_vocabulary=len(tokenizer.word_index) + 1 #+1 for padding\n",
    "print(size_of_vocabulary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ThBDgrP8JZnf"
   },
   "source": [
    "## Create embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3999,
     "status": "ok",
     "timestamp": 1586249687285,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "VdR43fJQJZng",
    "outputId": "9ee5aeef-bcee-4dfb-e474-87a37d3bdbbf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 24350 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "word_index = tokenizer.word_index\n",
    "print(\"Found %s unique tokens.\" % len(word_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uF9AT0LdJZnl"
   },
   "source": [
    "## Create the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 354
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5854,
     "status": "ok",
     "timestamp": 1586249689151,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "aUdRk0UDJZnl",
    "outputId": "5c9f6437-e27c-4539-eec1-3134fb5d52eb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 332, 300)          7305300   \n",
      "_________________________________________________________________\n",
      "lstm (LSTM)                  (None, 332, 128)          219648    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d (Global (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 7,533,269\n",
      "Trainable params: 7,533,269\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model=Sequential()\n",
    "\n",
    "#embedding layer\n",
    "model.add(Embedding(size_of_vocabulary,300,input_length=max_length,trainable=True)) \n",
    "\n",
    "#lstm layer\n",
    "model.add(LSTM(128,return_sequences=True,dropout=0.2))\n",
    "\n",
    "#Global Maxpooling\n",
    "model.add(GlobalMaxPooling1D())\n",
    "\n",
    "#Dense Layer\n",
    "model.add(Dense(64,activation='relu')) \n",
    "model.add(Dense(1,activation='relu')) \n",
    "\n",
    "#Add loss function, metrics, optimizer\n",
    "#optimizer = tf.keras.optimizers.RMSprop(0.001)\n",
    "\n",
    "model.compile(optimizer=\"RMSprop\", loss='mean_squared_error', metrics=[\"mae\"]) \n",
    "\n",
    "#Print summary of model\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hKZo_lzaJZnr"
   },
   "source": [
    "## Callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Kl98dyhkJZns"
   },
   "outputs": [],
   "source": [
    "earlystop = EarlyStopping(patience=10)\n",
    "learning_rate_reduction = ReduceLROnPlateau(monitor='val_mae', \n",
    "                                            patience=2, \n",
    "                                            verbose=1, \n",
    "                                            factor=0.5, \n",
    "                                            min_lr=0.00001)\n",
    "\n",
    "callbacks = [earlystop, learning_rate_reduction]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "op3MuPCKJZnx"
   },
   "source": [
    "## Fit the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 870
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 91363,
     "status": "ok",
     "timestamp": 1586249774680,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "_I34WFwPJZny",
    "outputId": "7e645fa5-a2fa-4876-929e-13e12e770f56"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n",
      "37/37 [==============================] - 4s 121ms/step - loss: 23.6479 - mae: 3.9662 - val_loss: 15.1412 - val_mae: 3.2900 - lr: 0.0010\n",
      "Epoch 2/1000\n",
      "37/37 [==============================] - 4s 111ms/step - loss: 14.9511 - mae: 3.2647 - val_loss: 14.3005 - val_mae: 3.1952 - lr: 0.0010\n",
      "Epoch 3/1000\n",
      "37/37 [==============================] - 4s 106ms/step - loss: 13.2553 - mae: 3.0810 - val_loss: 11.6766 - val_mae: 2.8670 - lr: 0.0010\n",
      "Epoch 4/1000\n",
      "37/37 [==============================] - 4s 108ms/step - loss: 9.2788 - mae: 2.5530 - val_loss: 7.3630 - val_mae: 2.1968 - lr: 0.0010\n",
      "Epoch 5/1000\n",
      "37/37 [==============================] - 4s 108ms/step - loss: 5.4450 - mae: 1.8891 - val_loss: 4.5940 - val_mae: 1.6993 - lr: 0.0010\n",
      "Epoch 6/1000\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 3.9306 - mae: 1.5637 - val_loss: 3.7002 - val_mae: 1.4841 - lr: 0.0010\n",
      "Epoch 7/1000\n",
      "37/37 [==============================] - 4s 109ms/step - loss: 3.1235 - mae: 1.3631 - val_loss: 3.3598 - val_mae: 1.4377 - lr: 0.0010\n",
      "Epoch 8/1000\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 2.3623 - mae: 1.1704 - val_loss: 4.0853 - val_mae: 1.6064 - lr: 0.0010\n",
      "Epoch 9/1000\n",
      "37/37 [==============================] - 4s 106ms/step - loss: 1.9470 - mae: 1.0819 - val_loss: 3.3066 - val_mae: 1.3788 - lr: 0.0010\n",
      "Epoch 10/1000\n",
      "37/37 [==============================] - 4s 109ms/step - loss: 1.5151 - mae: 0.9454 - val_loss: 3.2612 - val_mae: 1.3886 - lr: 0.0010\n",
      "Epoch 11/1000\n",
      "37/37 [==============================] - ETA: 0s - loss: 1.3146 - mae: 0.8897\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "37/37 [==============================] - 4s 108ms/step - loss: 1.3146 - mae: 0.8897 - val_loss: 4.0350 - val_mae: 1.5396 - lr: 0.0010\n",
      "Epoch 12/1000\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 0.8258 - mae: 0.6784 - val_loss: 3.4487 - val_mae: 1.3942 - lr: 5.0000e-04\n",
      "Epoch 13/1000\n",
      "37/37 [==============================] - 4s 106ms/step - loss: 0.6963 - mae: 0.6175 - val_loss: 3.2735 - val_mae: 1.3759 - lr: 5.0000e-04\n",
      "Epoch 14/1000\n",
      "37/37 [==============================] - 4s 108ms/step - loss: 0.5876 - mae: 0.5626 - val_loss: 3.3156 - val_mae: 1.3708 - lr: 5.0000e-04\n",
      "Epoch 15/1000\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 0.5562 - mae: 0.5457 - val_loss: 3.2809 - val_mae: 1.3766 - lr: 5.0000e-04\n",
      "Epoch 16/1000\n",
      "37/37 [==============================] - ETA: 0s - loss: 0.4794 - mae: 0.5171\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "37/37 [==============================] - 4s 108ms/step - loss: 0.4794 - mae: 0.5171 - val_loss: 3.3524 - val_mae: 1.3808 - lr: 5.0000e-04\n",
      "Epoch 17/1000\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 0.3279 - mae: 0.4030 - val_loss: 3.4143 - val_mae: 1.3903 - lr: 2.5000e-04\n",
      "Epoch 18/1000\n",
      "37/37 [==============================] - ETA: 0s - loss: 0.3046 - mae: 0.3847\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "37/37 [==============================] - 4s 108ms/step - loss: 0.3046 - mae: 0.3847 - val_loss: 3.3653 - val_mae: 1.4019 - lr: 2.5000e-04\n",
      "Epoch 19/1000\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 0.2552 - mae: 0.3360 - val_loss: 3.3960 - val_mae: 1.3959 - lr: 1.2500e-04\n",
      "Epoch 20/1000\n",
      "37/37 [==============================] - ETA: 0s - loss: 0.2364 - mae: 0.3242\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "37/37 [==============================] - 4s 108ms/step - loss: 0.2364 - mae: 0.3242 - val_loss: 3.3664 - val_mae: 1.4019 - lr: 1.2500e-04\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(np.array(x_tr_seq),\n",
    "                    np.array(y_tr),\n",
    "                    batch_size=128,\n",
    "                    epochs=1000,\n",
    "                    validation_data=(np.array(x_val_seq),np.array(y_val)),\n",
    "                    verbose=1,\n",
    "                    callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 92376,
     "status": "ok",
     "timestamp": 1586249775710,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "ojMu-A3wJZn4",
    "outputId": "0bcb623e-f0da-4944-cd0f-19e9285204c6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37/37 [==============================] - 0s 11ms/step - loss: 3.2862 - mae: 1.4019\n",
      "The val_mae is 1.402.\n"
     ]
    }
   ],
   "source": [
    "#evaluation \n",
    "val_loss, val_mae = model.evaluate(x_val_seq, y_val)\n",
    "\n",
    "print(\"The val_mae is %.3f.\" % val_mae)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 238
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 92378,
     "status": "error",
     "timestamp": 1586249775722,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "QX82za-8JZn_",
    "outputId": "b47bbdf8-c4e8-4524-93ba-6facb8efe5d7"
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-b9d575144017>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"loss\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"loss\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"val_loss\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"val_loss\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'loss'"
     ]
    }
   ],
   "source": [
    "plt.plot(model.history.history[\"loss\"], label=\"loss\");\n",
    "plt.plot(model.history.history[\"val_loss\"], label=\"val_loss\");\n",
    "plt.legend();\n",
    "plt.show();\n",
    "plt.close();\n",
    "\n",
    "plt.plot(model.history.history[\"mae\"], label=\"mae\");\n",
    "plt.plot(model.history.history[\"val_mae\"], label=\"val_mae\");\n",
    "plt.legend();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QSOY8AH2JZoF"
   },
   "source": [
    "## [Use Transfer Learning](https://www.analyticsvidhya.com/blog/2020/03/pretrained-word-embeddings-nlp/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 40110,
     "status": "ok",
     "timestamp": 1586249823321,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "GdKeDrv9JZoG",
    "outputId": "93f99b07-db89-4a16-d78d-f5aebb9387a3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "# load the whole embedding into memory\n",
    "embeddings_index = dict()\n",
    "\n",
    "with open(\"/content/drive/My Drive/Colab Notebooks/glove.6B/glove.6B.300d.txt\") as f:\n",
    "\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        coefs = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = coefs\n",
    "\n",
    "print('Loaded %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "dJqtmI0pJZoL"
   },
   "outputs": [],
   "source": [
    "# create a weight matrix for words in training docs\n",
    "embedding_matrix = np.zeros((size_of_vocabulary, 300))\n",
    "\n",
    "for word, i in tokenizer.word_index.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 354
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1412,
     "status": "ok",
     "timestamp": 1586249849820,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "NGM-hGE4JZoQ",
    "outputId": "f93b5d2f-98ae-48ab-d2e2-ebe809e126b8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 332, 300)          7305300   \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 332, 128)          219648    \n",
      "_________________________________________________________________\n",
      "global_max_pooling1d_2 (Glob (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 64)                8256      \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 7,533,269\n",
      "Trainable params: 7,533,269\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model2=Sequential()\n",
    "\n",
    "#embedding layer\n",
    "model2.add(Embedding(size_of_vocabulary,300,input_length=max_length,trainable=True)) \n",
    "\n",
    "#lstm layer\n",
    "model2.add(LSTM(128,return_sequences=True,dropout=0.2))\n",
    "\n",
    "#Global Maxpooling\n",
    "model2.add(GlobalMaxPooling1D())\n",
    "\n",
    "#Dense Layer\n",
    "model2.add(Dense(64,activation='relu')) \n",
    "model2.add(Dense(1,activation='relu')) \n",
    "\n",
    "#Add loss function, metrics, optimizer\n",
    "#optimizer = tf.keras.optimizers.RMSprop(0.001)\n",
    "\n",
    "model2.compile(optimizer=\"RMSprop\", loss='mean_squared_error', metrics=[\"mae\"]) \n",
    "\n",
    "#Print summary of model\n",
    "print(model2.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 798
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 72619,
     "status": "ok",
     "timestamp": 1586249923266,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "o4IlOKDvJZoU",
    "outputId": "78cb96c0-ac41-413f-ec2d-c13218a0ca03"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1000\n",
      "37/37 [==============================] - 5s 122ms/step - loss: 23.2733 - mae: 3.9171 - val_loss: 14.9484 - val_mae: 3.2747 - lr: 0.0010\n",
      "Epoch 2/1000\n",
      "37/37 [==============================] - 4s 106ms/step - loss: 14.7230 - mae: 3.2505 - val_loss: 13.6234 - val_mae: 3.1266 - lr: 0.0010\n",
      "Epoch 3/1000\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 10.3599 - mae: 2.7039 - val_loss: 5.6027 - val_mae: 1.9338 - lr: 0.0010\n",
      "Epoch 4/1000\n",
      "37/37 [==============================] - 4s 108ms/step - loss: 4.6331 - mae: 1.7053 - val_loss: 3.9897 - val_mae: 1.5779 - lr: 0.0010\n",
      "Epoch 5/1000\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 3.4195 - mae: 1.4152 - val_loss: 4.4526 - val_mae: 1.6776 - lr: 0.0010\n",
      "Epoch 6/1000\n",
      "37/37 [==============================] - 4s 106ms/step - loss: 2.8124 - mae: 1.2659 - val_loss: 3.6089 - val_mae: 1.4563 - lr: 0.0010\n",
      "Epoch 7/1000\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 2.1700 - mae: 1.1192 - val_loss: 3.2493 - val_mae: 1.4022 - lr: 0.0010\n",
      "Epoch 8/1000\n",
      "37/37 [==============================] - 4s 106ms/step - loss: 1.8999 - mae: 1.0501 - val_loss: 3.4722 - val_mae: 1.4487 - lr: 0.0010\n",
      "Epoch 9/1000\n",
      "37/37 [==============================] - ETA: 0s - loss: 1.7026 - mae: 0.9749\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.0005000000237487257.\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 1.7026 - mae: 0.9749 - val_loss: 5.4001 - val_mae: 1.8652 - lr: 0.0010\n",
      "Epoch 10/1000\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 1.1256 - mae: 0.7888 - val_loss: 3.4005 - val_mae: 1.4400 - lr: 5.0000e-04\n",
      "Epoch 11/1000\n",
      "37/37 [==============================] - ETA: 0s - loss: 0.9428 - mae: 0.7123\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.0002500000118743628.\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 0.9428 - mae: 0.7123 - val_loss: 3.5642 - val_mae: 1.4735 - lr: 5.0000e-04\n",
      "Epoch 12/1000\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 0.7293 - mae: 0.6078 - val_loss: 3.5418 - val_mae: 1.4835 - lr: 2.5000e-04\n",
      "Epoch 13/1000\n",
      "37/37 [==============================] - ETA: 0s - loss: 0.6682 - mae: 0.5786\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 0.0001250000059371814.\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 0.6682 - mae: 0.5786 - val_loss: 3.5332 - val_mae: 1.4742 - lr: 2.5000e-04\n",
      "Epoch 14/1000\n",
      "37/37 [==============================] - 4s 108ms/step - loss: 0.5882 - mae: 0.5235 - val_loss: 3.5172 - val_mae: 1.4742 - lr: 1.2500e-04\n",
      "Epoch 15/1000\n",
      "37/37 [==============================] - ETA: 0s - loss: 0.5508 - mae: 0.5059\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 6.25000029685907e-05.\n",
      "37/37 [==============================] - 4s 108ms/step - loss: 0.5508 - mae: 0.5059 - val_loss: 3.5208 - val_mae: 1.4684 - lr: 1.2500e-04\n",
      "Epoch 16/1000\n",
      "37/37 [==============================] - 4s 106ms/step - loss: 0.5245 - mae: 0.4838 - val_loss: 3.6241 - val_mae: 1.4956 - lr: 6.2500e-05\n",
      "Epoch 17/1000\n",
      "37/37 [==============================] - ETA: 0s - loss: 0.5129 - mae: 0.4754\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 3.125000148429535e-05.\n",
      "37/37 [==============================] - 4s 107ms/step - loss: 0.5129 - mae: 0.4754 - val_loss: 3.5785 - val_mae: 1.4883 - lr: 6.2500e-05\n"
     ]
    }
   ],
   "source": [
    "history = model2.fit(np.array(x_tr_seq),\n",
    "                    np.array(y_tr),\n",
    "                    batch_size=128,\n",
    "                    epochs=1000,\n",
    "                    validation_data=(np.array(x_val_seq),np.array(y_val)),\n",
    "                    verbose=1,\n",
    "                    callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 55731,
     "status": "ok",
     "timestamp": 1586249923651,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "88gF08odJZoY",
    "outputId": "56624259-b454-4293-d313-4932e91c90ff"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "37/37 [==============================] - 0s 12ms/step - loss: 3.7243 - mae: 1.4883\n",
      "The val_mae is 1.488.\n"
     ]
    }
   ],
   "source": [
    "#evaluation \n",
    "_, val_mae = model2.evaluate(x_val_seq, y_val)\n",
    "\n",
    "print(\"The val_mae is %.3f.\" % val_mae)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 107
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 61473,
     "status": "ok",
     "timestamp": 1586249930184,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "VjHaGr87fXHA",
    "outputId": "3a03e68c-2200-48f0-8d66-f78df4cb2f61"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:1817: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "If using Keras pass *_constraint arguments to layers.\n",
      "INFO:tensorflow:Assets written to: /content/drive/My Drive/final_project/lstm_pntk/assets\n"
     ]
    }
   ],
   "source": [
    "model2.save('/content/drive/My Drive/final_project/lstm_pntk')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 238
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 58591,
     "status": "error",
     "timestamp": 1586249930186,
     "user": {
      "displayName": "Evan Simpson",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gh4p47FxkA9aTnFpdt9dZI0LNlEYYizCC50unaC7Q=s64",
      "userId": "08573321944937795410"
     },
     "user_tz": -120
    },
    "id": "FY3cjzn3JZoe",
    "outputId": "c62739a4-ee22-418d-c643-236463307243"
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-28-3bad321a086e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"loss\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"loss\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"val_loss\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"val_loss\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlegend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshow\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m;\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'loss'"
     ]
    }
   ],
   "source": [
    "plt.plot(model2.history.history[\"loss\"], label=\"loss\");\n",
    "plt.plot(model2.history.history[\"val_loss\"], label=\"val_loss\");\n",
    "plt.legend();\n",
    "plt.show();\n",
    "plt.close();\n",
    "\n",
    "plt.plot(model2.history.history[\"mae\"], label=\"mae\");\n",
    "plt.plot(model2.history.history[\"val_mae\"], label=\"val_mae\");\n",
    "plt.legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3wPhkh6PPdSa"
   },
   "outputs": [],
   "source": [
    "a = model2.predict(x_val_seq)\n",
    "b = model2.predict(x_val_seq)\n",
    "\n",
    "csm_lstm_predictions_df = pd.DataFrame({\"pntk_custom\": list(a), \"pntk_transfer\": list(b)},\n",
    "                              columns = [\"pntk_custom\", \"pntk_transfer\"], \n",
    "                              index=test_data.index)\n",
    "\n",
    "csm_lstm_predictions_df.to_csv('/content/drive/My Drive/final_project/lstm_pntk.csv', index=\"False\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pfxICpfWuCpB"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "13_lstm_pntk.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}