hyperparameter-tuning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "import sklearn.decomposition as decomposition\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.metrics import f1_score, accuracy_score, make_scorer\n",
    "from sklearn.utils.class_weight import compute_sample_weight\n",
    "from sklearn.base import BaseEstimator\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras import Model, Input, optimizers\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.layers import LSTM, GRU, BatchNormalization\n",
    "from tensorflow.keras.models import load_model\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, LearningRateScheduler, ReduceLROnPlateau\n",
    "#%load_ext tensorboard\n",
    "#from tensorboard.plugins.hparams import api as hp\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sb\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "import matplotlib.patches as mpatches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((87554, 187, 1), (87554,))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv(\"data/mitbih_train.csv\", header=None)\n",
    "df_train = df_train.sample(frac=1)\n",
    "df_test = pd.read_csv(\"data/mitbih_test.csv\", header=None)\n",
    "\n",
    "Y = np.array(df_train[187].values).astype(np.int8)\n",
    "X = np.array(df_train[list(range(187))].values)[..., np.newaxis]\n",
    "\n",
    "Y_test = np.array(df_test[187].values).astype(np.int8)\n",
    "X_test = np.array(df_test[list(range(187))].values)[..., np.newaxis]\n",
    "\n",
    "n_class = np.unique(Y).size\n",
    "\n",
    "X.shape, Y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parameter search on RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_gru(n_class=5, dropout=0.3, rnn_sizes = [128, 128], fc_sizes=[64], batch_norm=True):\n",
    "    nclass = 5\n",
    "    model = Sequential()\n",
    "    model.add(Input(shape=(187, 1)))\n",
    "    \n",
    "    if batch_norm:\n",
    "        model.add(BatchNormalization())\n",
    "        \n",
    "    for index, dim in enumerate(rnn_sizes):\n",
    "        model.add(GRU(dim, dropout=dropout, return_sequences=(index != len(rnn_sizes) - 1)))\n",
    "        \n",
    "        if batch_norm:\n",
    "            model.add(BatchNormalization())\n",
    "    \n",
    "    for index, dim in enumerate(fc_sizes):\n",
    "        model.add(Dense(dim, activation=\"relu\"))\n",
    "        model.add(Dropout(dropout))\n",
    "        \n",
    "        if batch_norm:\n",
    "            model.add(BatchNormalization())\n",
    "            \n",
    "    model.add(Dense(nclass, activation=\"softmax\"))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomRNN(BaseEstimator):\n",
    "    def fit(self, train_X, train_y, **kwargs):\n",
    "        \n",
    "        self.build_model()\n",
    "        \n",
    "        # early = EarlyStopping(monitor=\"val_accuracy\", mode=\"max\", patience=5, verbose=1)\n",
    "        # redonplat = ReduceLROnPlateau(monitor=\"val_accuracy\", mode=\"max\", patience=3, verbose=2)\n",
    "        # callbacks_list = [checkpoint, early, redonplat]  # early\n",
    "        self.model.fit(train_X, train_y, validation_split=0.1, epochs=self.epochs, batch_size=self.batch_size)\n",
    "    \n",
    "    def predict(self, eval_X):\n",
    "        return np.argmax(self.model.predict(eval_X), axis=1)\n",
    "    \n",
    "    def set_params(self, epochs=100, \n",
    "                         batch_size=64, \n",
    "                         learning_rate=1e-3, \n",
    "                         dropout=0.3, \n",
    "                         rnn_sizes=[128, 128],\n",
    "                         fc_sizes=[64],\n",
    "                         batch_norm=True):\n",
    "        self.epochs = epochs\n",
    "        self.batch_size = batch_size\n",
    "        self.learning_rate = learning_rate\n",
    "        self.dropout = dropout\n",
    "        self.rnn_sizes = rnn_sizes\n",
    "        self.fc_sizes = fc_sizes\n",
    "        self.batch_norm = batch_norm\n",
    "                \n",
    "        return self\n",
    "    \n",
    "    def score(self, eval_X, eval_y):\n",
    "        predicted_y = np.argmax(self.model.predict(eval_X), axis=1)\n",
    "        f1_score_ = f1_score(predicted_y, eval_y, average='macro')\n",
    "        print(\"f1 score: \", f1_score_)\n",
    "        return f1_score_\n",
    "        \n",
    "    def build_model(self):\n",
    "        self.model = build_gru(n_class=5, dropout=self.dropout, \n",
    "                          rnn_sizes=self.rnn_sizes, fc_sizes=self.fc_sizes, batch_norm=self.batch_norm)\n",
    "        opt = optimizers.Adam(self.learning_rate)\n",
    "            \n",
    "        self.model.compile(optimizer=opt, \n",
    "                      loss=\"sparse_categorical_crossentropy\", \n",
    "                      metrics=['accuracy'])        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "[CV] batch_norm=False, batch_size=128, dropout=0.2, epochs=10, fc_sizes=[64, 32], learning_rate=0.0001, rnn_sizes=[128, 128] \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 63038 samples, validate on 7005 samples\n",
      "Epoch 1/10\n",
      "63038/63038 [==============================] - 17s 264us/sample - loss: 0.8070 - accuracy: 0.8274 - val_loss: 0.6418 - val_accuracy: 0.8344\n",
      "Epoch 2/10\n",
      "63038/63038 [==============================] - 12s 194us/sample - loss: 0.6866 - accuracy: 0.8280 - val_loss: 0.6440 - val_accuracy: 0.8348\n",
      "Epoch 3/10\n",
      "63038/63038 [==============================] - 12s 194us/sample - loss: 0.6785 - accuracy: 0.8284 - val_loss: 0.6386 - val_accuracy: 0.8350\n",
      "Epoch 4/10\n",
      "63038/63038 [==============================] - 12s 192us/sample - loss: 0.6751 - accuracy: 0.8285 - val_loss: 0.6378 - val_accuracy: 0.8350\n",
      "Epoch 5/10\n",
      "63038/63038 [==============================] - 12s 192us/sample - loss: 0.6696 - accuracy: 0.8283 - val_loss: 0.6372 - val_accuracy: 0.8357\n",
      "Epoch 6/10\n",
      "63038/63038 [==============================] - 12s 195us/sample - loss: 0.6668 - accuracy: 0.8282 - val_loss: 0.6371 - val_accuracy: 0.8355\n",
      "Epoch 7/10\n",
      "63038/63038 [==============================] - 13s 199us/sample - loss: 0.6655 - accuracy: 0.8283 - val_loss: 0.6356 - val_accuracy: 0.8357\n",
      "Epoch 8/10\n",
      "63038/63038 [==============================] - 13s 202us/sample - loss: 0.6599 - accuracy: 0.8282 - val_loss: 0.6337 - val_accuracy: 0.8357\n",
      "Epoch 9/10\n",
      "63038/63038 [==============================] - 12s 198us/sample - loss: 0.6595 - accuracy: 0.8278 - val_loss: 0.6294 - val_accuracy: 0.8354\n",
      "Epoch 10/10\n",
      "63038/63038 [==============================] - 12s 197us/sample - loss: 0.6564 - accuracy: 0.8277 - val_loss: 0.6266 - val_accuracy: 0.8353\n",
      "f1 score:  0.19406401492751407\n",
      "f1 score:  0.19105538931503693\n",
      "[CV]  batch_norm=False, batch_size=128, dropout=0.2, epochs=10, fc_sizes=[64, 32], learning_rate=0.0001, rnn_sizes=[128, 128], score=(train=0.191, test=0.194), total= 2.2min\n",
      "[CV] batch_norm=False, batch_size=128, dropout=0.2, epochs=10, fc_sizes=[64, 32], learning_rate=0.0001, rnn_sizes=[128, 128] \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  2.4min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 63038 samples, validate on 7005 samples\n",
      "Epoch 1/10\n",
      "63038/63038 [==============================] - 15s 233us/sample - loss: 0.8186 - accuracy: 0.8244\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 3331, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-10-36069508b718>\", line 30, in <module>\n",
      "    best = search.fit(X[:, :, :], Y[:])\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\sklearn\\model_selection\\_search.py\", line 710, in fit\n",
      "    self._run_search(evaluate_candidates)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\sklearn\\model_selection\\_search.py\", line 1151, in _run_search\n",
      "    evaluate_candidates(ParameterGrid(self.param_grid))\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\sklearn\\model_selection\\_search.py\", line 689, in evaluate_candidates\n",
      "    cv.split(X, y, groups)))\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\joblib\\parallel.py\", line 1007, in __call__\n",
      "    while self.dispatch_one_batch(iterator):\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\joblib\\parallel.py\", line 835, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\joblib\\parallel.py\", line 754, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 209, in apply_async\n",
      "    result = ImmediateResult(func)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 590, in __init__\n",
      "    self.results = batch()\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\joblib\\parallel.py\", line 256, in __call__\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\joblib\\parallel.py\", line 256, in <listcomp>\n",
      "    for func, args, kwargs in self.items]\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\sklearn\\model_selection\\_validation.py\", line 515, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"<ipython-input-9-0e30a8878e43>\", line 9, in fit\n",
      "    self.model.fit(train_X, train_y, validation_split=0.1, epochs=self.epochs, batch_size=self.batch_size)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training.py\", line 819, in fit\n",
      "    use_multiprocessing=use_multiprocessing)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_v2.py\", line 395, in fit\n",
      "    total_epochs=1)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_v2.py\", line 128, in run_one_epoch\n",
      "    batch_outs = execution_function(iterator)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_v2_utils.py\", line 98, in execution_function\n",
      "    distributed_function(input_fn))\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\eager\\def_function.py\", line 568, in __call__\n",
      "    result = self._call(*args, **kwds)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\eager\\def_function.py\", line 615, in _call\n",
      "    self._initialize(args, kwds, add_initializers_to=initializers)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\eager\\def_function.py\", line 497, in _initialize\n",
      "    *args, **kwds))\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\", line 2389, in _get_concrete_function_internal_garbage_collected\n",
      "    graph_function, _, _ = self._maybe_define_function(args, kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\", line 2703, in _maybe_define_function\n",
      "    graph_function = self._create_graph_function(args, kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\", line 2593, in _create_graph_function\n",
      "    capture_by_value=self._capture_by_value),\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\framework\\func_graph.py\", line 978, in func_graph_from_py_func\n",
      "    func_outputs = python_func(*func_args, **func_kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\eager\\def_function.py\", line 439, in wrapped_fn\n",
      "    return weak_wrapped_fn().__wrapped__(*args, **kwds)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_v2_utils.py\", line 85, in distributed_function\n",
      "    per_replica_function, args=args)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\distribute\\distribute_lib.py\", line 763, in experimental_run_v2\n",
      "    return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\distribute\\distribute_lib.py\", line 1819, in call_for_each_replica\n",
      "    return self._call_for_each_replica(fn, args, kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\distribute\\distribute_lib.py\", line 2164, in _call_for_each_replica\n",
      "    return fn(*args, **kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\autograph\\impl\\api.py\", line 292, in wrapper\n",
      "    return func(*args, **kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_v2_utils.py\", line 505, in test_on_batch\n",
      "    output_loss_metrics=model._output_loss_metrics)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_eager.py\", line 355, in test_on_batch\n",
      "    output_loss_metrics=output_loss_metrics))\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\training_eager.py\", line 127, in _model_loss\n",
      "    outs = model(inputs, **kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\base_layer.py\", line 778, in __call__\n",
      "    outputs = call_fn(cast_inputs, *args, **kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\sequential.py\", line 267, in call\n",
      "    return super(Sequential, self).call(inputs, training=training, mask=mask)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\network.py\", line 717, in call\n",
      "    convert_kwargs_to_constants=base_layer_utils.call_context().saving)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\network.py\", line 891, in _run_internal_graph\n",
      "    output_tensors = layer(computed_tensors, **kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\layers\\recurrent.py\", line 644, in __call__\n",
      "    return super(RNN, self).__call__(inputs, **kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\engine\\base_layer.py\", line 778, in __call__\n",
      "    outputs = call_fn(cast_inputs, *args, **kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\layers\\recurrent_v2.py\", line 422, in call\n",
      "    inputs, initial_state, training, mask, row_lengths)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\layers\\recurrent_v2.py\", line 483, in _defun_gru_call\n",
      "    **normal_gru_kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\keras\\layers\\recurrent_v2.py\", line 764, in gru_with_backend_selection\n",
      "    function.register(defun_cudnn_gru, **params)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\", line 2731, in register\n",
      "    concrete_func.add_gradient_functions_to_graph()\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\", line 1821, in add_gradient_functions_to_graph\n",
      "    self._delayed_rewrite_functions.forward_backward())\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\", line 616, in forward_backward\n",
      "    forward, backward = self._construct_forward_backward(num_doutputs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\", line 702, in _construct_forward_backward\n",
      "    self._func_graph.outputs, forward_function_attr)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\tensorflow_core\\python\\eager\\function.py\", line 451, in __init__\n",
      "    compat.as_str(\"\"))\n",
      "KeyboardInterrupt\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2044, in showtraceback\n",
      "    stb = value._render_traceback_()\n",
      "AttributeError: 'KeyboardInterrupt' object has no attribute '_render_traceback_'\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 1151, in get_records\n",
      "    return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 319, in wrapped\n",
      "    return f(*args, **kwargs)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\site-packages\\IPython\\core\\ultratb.py\", line 353, in _fixed_getinnerframes\n",
      "    records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\inspect.py\", line 1502, in getinnerframes\n",
      "    frameinfo = (tb.tb_frame,) + getframeinfo(tb, context)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\inspect.py\", line 1460, in getframeinfo\n",
      "    filename = getsourcefile(frame) or getfile(frame)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\inspect.py\", line 696, in getsourcefile\n",
      "    if getattr(getmodule(object, filename), '__loader__', None) is not None:\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\inspect.py\", line 739, in getmodule\n",
      "    f = getabsfile(module)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\inspect.py\", line 708, in getabsfile\n",
      "    _filename = getsourcefile(object) or getfile(object)\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\inspect.py\", line 693, in getsourcefile\n",
      "    if os.path.exists(filename):\n",
      "  File \"C:\\Users\\User\\Anaconda3\\envs\\tf-gpu\\lib\\genericpath.py\", line 19, in exists\n",
      "    os.stat(path)\n",
      "KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m"
     ]
    }
   ],
   "source": [
    "params = {\n",
    "    'epochs': [2],\n",
    "    'batch_size': [64],\n",
    "    'learning_rate': [1e-3],\n",
    "    'dropout': [0.2],\n",
    "    'rnn_sizes': [[128, 128], [128, 128, 128]],\n",
    "    'fc_sizes': [[64], [64, 64], [64, 32]],\n",
    "    'batch_norm': [True, False]\n",
    "}\n",
    "\n",
    "dummy_params = {\n",
    "    'epochs': [10],\n",
    "    'batch_size': [128],\n",
    "    'learning_rate': [1e-4],\n",
    "    'dropout': [0.2],\n",
    "    'rnn_sizes': [[128, 128]],\n",
    "    'fc_sizes': [[64, 32]],\n",
    "    'batch_norm': [False]\n",
    "}\n",
    "\n",
    "model = CustomRNN()\n",
    "search = GridSearchCV(estimator=model, \n",
    "                      param_grid=dummy_params,\n",
    "                      n_jobs=1,\n",
    "                      cv=5,\n",
    "                      return_train_score=True, \n",
    "                      refit=False, \n",
    "                      verbose=10,\n",
    "                      error_score='raise')\n",
    "best = search.fit(X[:, :, :], Y[:])\n",
    "best.__dict__"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}