Tensoboard-tfds-IMDB.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb.npz\n",
      "17465344/17464789 [==============================] - 14s 1us/step\n",
      "25000 Training sequences\n",
      "25000 Validation sequences\n"
     ]
    }
   ],
   "source": [
    "import tensorflow\n",
    "from tensorflow import keras\n",
    "max_features=100000\n",
    "(train_x, train_y), (test_x, test_y) = keras.datasets.imdb.load_data(\n",
    "    num_words=max_features\n",
    ")\n",
    "print(len(train_x), \"Training sequences\")\n",
    "print(len(test_x), \"Validation sequences\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 2.54 ms\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "import tensorboard\n",
    "%load_ext tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "maxlen=100\n",
    "train_x = keras.preprocessing.sequence.pad_sequences(train_x, maxlen=maxlen)\n",
    "test_x = keras.preprocessing.sequence.pad_sequences(test_x, maxlen=maxlen)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: MiniAttention==0.1 in d:\\anaconda\\lib\\site-packages (0.1)\n",
      "Requirement already satisfied: keras in d:\\anaconda\\lib\\site-packages (from MiniAttention==0.1) (2.4.3)\n",
      "Requirement already satisfied: tensorflow in d:\\anaconda\\lib\\site-packages (from MiniAttention==0.1) (2.3.1)\n",
      "Requirement already satisfied: numpy in d:\\anaconda\\lib\\site-packages (from MiniAttention==0.1) (1.18.5)\n",
      "Requirement already satisfied: pandas in d:\\anaconda\\lib\\site-packages (from MiniAttention==0.1) (1.0.5)\n",
      "Requirement already satisfied: pyyaml in d:\\anaconda\\lib\\site-packages (from keras->MiniAttention==0.1) (5.3.1)\n",
      "Requirement already satisfied: h5py in d:\\anaconda\\lib\\site-packages (from keras->MiniAttention==0.1) (2.10.0)\n",
      "Requirement already satisfied: scipy>=0.14 in d:\\anaconda\\lib\\site-packages (from keras->MiniAttention==0.1) (1.5.0)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (1.1.0)\n",
      "Requirement already satisfied: keras-preprocessing<1.2,>=1.1.1 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (1.1.2)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (1.32.0)\n",
      "Requirement already satisfied: gast==0.3.3 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (0.3.3)\n",
      "Requirement already satisfied: astunparse==1.6.3 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (1.6.3)\n",
      "Requirement already satisfied: six>=1.12.0 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (1.15.0)\n",
      "Requirement already satisfied: google-pasta>=0.1.8 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (0.2.0)\n",
      "Requirement already satisfied: tensorboard<3,>=2.3.0 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (2.3.0)\n",
      "Requirement already satisfied: tensorflow-estimator<2.4.0,>=2.3.0 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (2.3.0)\n",
      "Requirement already satisfied: wrapt>=1.11.1 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (1.11.2)\n",
      "Requirement already satisfied: wheel>=0.26 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (0.34.2)\n",
      "Requirement already satisfied: protobuf>=3.9.2 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (3.13.0)\n",
      "Requirement already satisfied: opt-einsum>=2.3.2 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (3.3.0)\n",
      "Requirement already satisfied: absl-py>=0.7.0 in d:\\anaconda\\lib\\site-packages (from tensorflow->MiniAttention==0.1) (0.10.0)\n",
      "Requirement already satisfied: python-dateutil>=2.6.1 in d:\\anaconda\\lib\\site-packages (from pandas->MiniAttention==0.1) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in d:\\anaconda\\lib\\site-packages (from pandas->MiniAttention==0.1) (2020.1)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in d:\\anaconda\\lib\\site-packages (from tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (1.21.3)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in d:\\anaconda\\lib\\site-packages (from tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (1.0.1)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in d:\\anaconda\\lib\\site-packages (from tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (1.7.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in d:\\anaconda\\lib\\site-packages (from tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (3.2.2)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in d:\\anaconda\\lib\\site-packages (from tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (0.4.1)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in d:\\anaconda\\lib\\site-packages (from tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (49.2.0.post20200714)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in d:\\anaconda\\lib\\site-packages (from tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (2.24.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in d:\\anaconda\\lib\\site-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (0.2.8)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in d:\\anaconda\\lib\\site-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (4.1.1)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.5\" in d:\\anaconda\\lib\\site-packages (from google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (4.6)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in d:\\anaconda\\lib\\site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (1.3.0)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in d:\\anaconda\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in d:\\anaconda\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (2.10)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in d:\\anaconda\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (1.25.9)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in d:\\anaconda\\lib\\site-packages (from requests<3,>=2.21.0->tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (2020.6.20)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in d:\\anaconda\\lib\\site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in d:\\anaconda\\lib\\site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<3,>=2.3.0->tensorflow->MiniAttention==0.1) (3.1.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install MiniAttention==0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"functional_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         [(None, 72)]              0         \n",
      "_________________________________________________________________\n",
      "embedding_1 (Embedding)      (None, 72, 256)           25600000  \n",
      "_________________________________________________________________\n",
      "mini_attention_block_2 (Mini (None, 72, 256)           0         \n",
      "_________________________________________________________________\n",
      "bidirectional_2 (Bidirection (None, 72, 256)           394240    \n",
      "_________________________________________________________________\n",
      "bidirectional_3 (Bidirection (None, 72, 128)           164352    \n",
      "_________________________________________________________________\n",
      "mini_attention_block_3 (Mini (None, 72, 128)           0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 72, 64)            8256      \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 72, 64)            4160      \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 72, 1)             65        \n",
      "=================================================================\n",
      "Total params: 26,171,073\n",
      "Trainable params: 26,171,073\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "WARNING:tensorflow:Model failed to serialize as JSON. Ignoring... Layer MiniAttentionBlock has arguments in `__init__` and therefore must override `get_config`.\n",
      "Epoch 1/10\n",
      "WARNING:tensorflow:Model was constructed with shape (None, 72) for input Tensor(\"input_2:0\", shape=(None, 72), dtype=float32), but it was called on an input with incompatible shape (None, 100).\n",
      "WARNING:tensorflow:Model was constructed with shape (None, 72) for input Tensor(\"input_2:0\", shape=(None, 72), dtype=float32), but it was called on an input with incompatible shape (None, 100).\n",
      "   2/1563 [..............................] - ETA: 57:52 - loss: 0.6931 - accuracy: 0.5938WARNING:tensorflow:Callbacks method `on_train_batch_end` is slow compared to the batch time (batch time: 0.1038s vs `on_train_batch_end` time: 4.3457s). Check your callbacks.\n",
      "1563/1563 [==============================] - ETA: 0s - loss: 0.6932 - accuracy: 0.4999WARNING:tensorflow:Model was constructed with shape (None, 72) for input Tensor(\"input_2:0\", shape=(None, 72), dtype=float32), but it was called on an input with incompatible shape (None, 100).\n",
      "1563/1563 [==============================] - 235s 150ms/step - loss: 0.6932 - accuracy: 0.4999 - val_loss: 0.6931 - val_accuracy: 0.5000\n",
      "Epoch 2/10\n",
      "1563/1563 [==============================] - 245s 157ms/step - loss: 0.6932 - accuracy: 0.5000 - val_loss: 0.6931 - val_accuracy: 0.5000\n",
      "Epoch 3/10\n",
      "1563/1563 [==============================] - 247s 158ms/step - loss: 0.6932 - accuracy: 0.5000 - val_loss: 0.6931 - val_accuracy: 0.5000\n",
      "Epoch 4/10\n",
      "1563/1563 [==============================] - 243s 156ms/step - loss: 0.6932 - accuracy: 0.5000 - val_loss: 0.6931 - val_accuracy: 0.5000\n",
      "Epoch 5/10\n",
      "1563/1563 [==============================] - 251s 161ms/step - loss: 0.6932 - accuracy: 0.5000 - val_loss: 0.6932 - val_accuracy: 0.5000\n",
      "Epoch 6/10\n",
      "1563/1563 [==============================] - 273s 175ms/step - loss: 0.6932 - accuracy: 0.5000 - val_loss: 0.6931 - val_accuracy: 0.5000\n",
      "Epoch 7/10\n",
      "1563/1563 [==============================] - 263s 169ms/step - loss: 0.6931 - accuracy: 0.5000 - val_loss: 0.6931 - val_accuracy: 0.5000\n",
      "Epoch 8/10\n",
      "1563/1563 [==============================] - 270s 173ms/step - loss: 0.6931 - accuracy: 0.5000 - val_loss: 0.6931 - val_accuracy: 0.5000\n",
      "Epoch 9/10\n",
      "1563/1563 [==============================] - 276s 176ms/step - loss: 0.6932 - accuracy: 0.5000 - val_loss: 0.6931 - val_accuracy: 0.5000\n",
      "Epoch 10/10\n",
      "1563/1563 [==============================] - 287s 184ms/step - loss: 0.6932 - accuracy: 0.5000 - val_loss: 0.6931 - val_accuracy: 0.5000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Reusing TensorBoard on port 6006 (pid 27092), started 0:46:00 ago. (Use '!kill 27092' to kill it.)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-5b8f57332a519217\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-5b8f57332a519217\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          const port = 6006;\n",
       "          if (port) {\n",
       "            url.port = port;\n",
       "          }\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import MiniAttention.MiniAttention as ma\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from keras import backend as k\n",
    "from keras.layers import LSTM,Dense,Flatten,Bidirectional\n",
    "from keras.activations import softmax,relu,elu,sigmoid\n",
    "from keras.optimizers import Adagrad\n",
    "from keras.initializers import glorot_uniform   \n",
    "from keras.regularizers import l2\n",
    "from keras.constraints import min_max_norm\n",
    "from keras.layers import Embedding,Input\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Layer\n",
    "import os\n",
    "import datetime\n",
    "\n",
    "def network(inp_shape,max_features,out_shape,train_x,train_y,test_x,test_y):\n",
    "    #embedding_map=create_glove_embedding(train_x)\n",
    "    #emb_mat,word_idx,emb_dim=embedding_preprocess(embedding_map,max_features,inp_shape,train_x,train_y,test_x,test_y)\n",
    "    inp=Input(shape=(inp_shape,))\n",
    "    #freeze emb weights\n",
    "    #z=tf.keras.layers.Embedding(word_idx, emb_dim, weights=[emb_mat], trainable=False)(inp)\n",
    "    z=Embedding(max_features,256)(inp)\n",
    "    \n",
    "    z=ma.MiniAttentionBlock(keras.initializers.he_uniform,None,None,keras.regularizers.L2(l2=0.02),None,None,None,None,None)(z)\n",
    "    \n",
    "    z=tf.keras.layers.Bidirectional(LSTM(128,recurrent_activation=\"relu\",return_sequences=True))(z)\n",
    "    z=tf.keras.layers.Bidirectional(LSTM(64,recurrent_activation=\"relu\",return_sequences=True))(z)\n",
    "    z=ma.MiniAttentionBlock(keras.initializers.he_uniform,None,None,keras.regularizers.L2(l2=0.02),None,None,None,None,None)(z)\n",
    "    z=keras.layers.Dense(64,activation=\"relu\")(z)\n",
    "    z=keras.layers.Dense(64,activation=\"relu\")(z)\n",
    "    z=keras.layers.Dense(1,activation=\"sigmoid\")(z)\n",
    "    model=keras.models.Model(inputs=inp,outputs=z)\n",
    "    model.compile(loss=\"binary_crossentropy\",metrics=['accuracy'],optimizer=keras.optimizers.Adagrad(learning_rate=1e-3))\n",
    "    model.summary()\n",
    "    return model\n",
    "\n",
    "\n",
    "logdir = os.path.join(\"logs\", datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\"))\n",
    "tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=1)\n",
    "\n",
    "\n",
    "max_features=100000\n",
    "emb_size=300\n",
    "inp_shape=72\n",
    "out_shape=1\n",
    "maxlen=100\n",
    "model=network(inp_shape,max_features,out_shape,train_x,train_y,test_x,test_y)  \n",
    "model.fit(train_x,train_y,epochs=10,batch_size=16,verbose=1,validation_data=(test_x,test_y),callbacks=[tensorboard_callback])\n",
    "%tensorboard --logdir logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Reusing TensorBoard on port 6006 (pid 27092), started 0:01:54 ago. (Use '!kill 27092' to kill it.)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-d1991f410ca3ca9f\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-d1991f410ca3ca9f\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          const port = 6006;\n",
       "          if (port) {\n",
       "            url.port = port;\n",
       "          }\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%tensorboard --logdir logs"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}