10_ANNs_keras.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow import keras\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "fashion_mnist = keras.datasets.fashion_mnist\n",
    "(X_train_full, y_train_full), (X_test, y_test) = fashion_mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_valid, X_train = X_train_full[:5000] / 255.0, X_train_full[5000:] / 255.0\n",
    "y_valid, y_train = y_train_full[:5000], y_train_full[5000:]\n",
    "X_test = X_test / 255.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_names = [\"T-shirt/top\", \"Trouser\", \"Pullover\", \"Dress\", \"Coat\",\n",
    "               \"Sandal\", \"Shirt\", \"Sneaker\", \"Bag\", \"Ankle boot\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Coat'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class_names[y_train[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-01-23 10:50:34.099326: I tensorflow/compiler/jit/xla_cpu_device.cc:41] Not creating XLA devices, tf_xla_enable_xla_devices not set\n",
      "2022-01-23 10:50:34.100603: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  SSE4.1 SSE4.2 AVX AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-01-23 10:50:34.103333: I tensorflow/core/common_runtime/process_util.cc:146] Creating new thread pool with default inter op setting: 2. Tune using inter_op_parallelism_threads for best performance.\n"
     ]
    }
   ],
   "source": [
    "model = keras.models.Sequential()\n",
    "model.add(keras.layers.Flatten(input_shape=[28, 28]))\n",
    "model.add(keras.layers.Dense(300, activation=\"relu\"))\n",
    "model.add(keras.layers.Dense(100, activation=\"relu\"))\n",
    "model.add(keras.layers.Dense(10, activation=\"softmax\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten (Flatten)            (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 300)               235500    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                1010      \n",
      "=================================================================\n",
      "Total params: 266,610\n",
      "Trainable params: 266,610\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tensorflow.python.keras.layers.core.Flatten at 0x7feb3ecfffd0>,\n",
       " <tensorflow.python.keras.layers.core.Dense at 0x7feb3eaaa908>,\n",
       " <tensorflow.python.keras.layers.core.Dense at 0x7feb3ea97be0>,\n",
       " <tensorflow.python.keras.layers.core.Dense at 0x7feb3f3e2e48>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'dense'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.layers[1].name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.layers.core.Flatten at 0x7feb3ecfffd0>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.get_layer('flatten')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden1 = model.layers[1]\n",
    "weights, biases = hidden1.get_weights()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss=keras.losses.sparse_categorical_crossentropy,\n",
    "              optimizer=keras.optimizers.SGD(),\n",
    "              metrics=keras.metrics.sparse_categorical_accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-01-23 13:28:51.832481: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)\n",
      "2022-01-23 13:28:51.857079: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2099940000 Hz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "1719/1719 [==============================] - 17s 9ms/step - loss: 0.9829 - sparse_categorical_accuracy: 0.6891 - val_loss: 0.5291 - val_sparse_categorical_accuracy: 0.8182\n",
      "Epoch 2/30\n",
      "1719/1719 [==============================] - 24s 14ms/step - loss: 0.4957 - sparse_categorical_accuracy: 0.8282 - val_loss: 0.4413 - val_sparse_categorical_accuracy: 0.8514\n",
      "Epoch 3/30\n",
      "1719/1719 [==============================] - 25s 15ms/step - loss: 0.4455 - sparse_categorical_accuracy: 0.8430 - val_loss: 0.4175 - val_sparse_categorical_accuracy: 0.8582\n",
      "Epoch 4/30\n",
      "1719/1719 [==============================] - 35s 21ms/step - loss: 0.4173 - sparse_categorical_accuracy: 0.8542 - val_loss: 0.3894 - val_sparse_categorical_accuracy: 0.8664\n",
      "Epoch 5/30\n",
      "1719/1719 [==============================] - 20s 12ms/step - loss: 0.3997 - sparse_categorical_accuracy: 0.8606 - val_loss: 0.3888 - val_sparse_categorical_accuracy: 0.8680\n",
      "Epoch 6/30\n",
      "1719/1719 [==============================] - 34s 20ms/step - loss: 0.3813 - sparse_categorical_accuracy: 0.8647 - val_loss: 0.3829 - val_sparse_categorical_accuracy: 0.8628\n",
      "Epoch 7/30\n",
      "1719/1719 [==============================] - 34s 20ms/step - loss: 0.3634 - sparse_categorical_accuracy: 0.8702 - val_loss: 0.3568 - val_sparse_categorical_accuracy: 0.8756\n",
      "Epoch 8/30\n",
      "1719/1719 [==============================] - 32s 19ms/step - loss: 0.3537 - sparse_categorical_accuracy: 0.8752 - val_loss: 0.3532 - val_sparse_categorical_accuracy: 0.8764\n",
      "Epoch 9/30\n",
      "1719/1719 [==============================] - 35s 20ms/step - loss: 0.3413 - sparse_categorical_accuracy: 0.8777 - val_loss: 0.3397 - val_sparse_categorical_accuracy: 0.8790\n",
      "Epoch 10/30\n",
      "1719/1719 [==============================] - 40s 23ms/step - loss: 0.3312 - sparse_categorical_accuracy: 0.8814 - val_loss: 0.3451 - val_sparse_categorical_accuracy: 0.8790\n",
      "Epoch 11/30\n",
      "1719/1719 [==============================] - 36s 21ms/step - loss: 0.3237 - sparse_categorical_accuracy: 0.8844 - val_loss: 0.3271 - val_sparse_categorical_accuracy: 0.8812\n",
      "Epoch 12/30\n",
      "1719/1719 [==============================] - 42s 24ms/step - loss: 0.3143 - sparse_categorical_accuracy: 0.8882 - val_loss: 0.3375 - val_sparse_categorical_accuracy: 0.8812\n",
      "Epoch 13/30\n",
      "1719/1719 [==============================] - 45s 26ms/step - loss: 0.3078 - sparse_categorical_accuracy: 0.8905 - val_loss: 0.3233 - val_sparse_categorical_accuracy: 0.8848\n",
      "Epoch 14/30\n",
      "1719/1719 [==============================] - 35s 21ms/step - loss: 0.3073 - sparse_categorical_accuracy: 0.8900 - val_loss: 0.3166 - val_sparse_categorical_accuracy: 0.8884\n",
      "Epoch 15/30\n",
      "1719/1719 [==============================] - 30s 17ms/step - loss: 0.2918 - sparse_categorical_accuracy: 0.8935 - val_loss: 0.3138 - val_sparse_categorical_accuracy: 0.8912\n",
      "Epoch 16/30\n",
      "1719/1719 [==============================] - 34s 20ms/step - loss: 0.2842 - sparse_categorical_accuracy: 0.8978 - val_loss: 0.3449 - val_sparse_categorical_accuracy: 0.8790\n",
      "Epoch 17/30\n",
      "1719/1719 [==============================] - 33s 19ms/step - loss: 0.2798 - sparse_categorical_accuracy: 0.9006 - val_loss: 0.3201 - val_sparse_categorical_accuracy: 0.8858\n",
      "Epoch 18/30\n",
      "1719/1719 [==============================] - 33s 19ms/step - loss: 0.2784 - sparse_categorical_accuracy: 0.9004 - val_loss: 0.3101 - val_sparse_categorical_accuracy: 0.8910\n",
      "Epoch 19/30\n",
      "1719/1719 [==============================] - 28s 16ms/step - loss: 0.2718 - sparse_categorical_accuracy: 0.9015 - val_loss: 0.3062 - val_sparse_categorical_accuracy: 0.8912\n",
      "Epoch 20/30\n",
      "1719/1719 [==============================] - 32s 19ms/step - loss: 0.2611 - sparse_categorical_accuracy: 0.9049 - val_loss: 0.3312 - val_sparse_categorical_accuracy: 0.8804\n",
      "Epoch 21/30\n",
      "1719/1719 [==============================] - 27s 16ms/step - loss: 0.2647 - sparse_categorical_accuracy: 0.9047 - val_loss: 0.3037 - val_sparse_categorical_accuracy: 0.8902\n",
      "Epoch 22/30\n",
      "1719/1719 [==============================] - 27s 16ms/step - loss: 0.2545 - sparse_categorical_accuracy: 0.9088 - val_loss: 0.3020 - val_sparse_categorical_accuracy: 0.8908\n",
      "Epoch 23/30\n",
      "1719/1719 [==============================] - 30s 17ms/step - loss: 0.2450 - sparse_categorical_accuracy: 0.9121 - val_loss: 0.3098 - val_sparse_categorical_accuracy: 0.8872\n",
      "Epoch 24/30\n",
      "1719/1719 [==============================] - 32s 18ms/step - loss: 0.2458 - sparse_categorical_accuracy: 0.9108 - val_loss: 0.3037 - val_sparse_categorical_accuracy: 0.8906\n",
      "Epoch 25/30\n",
      "1719/1719 [==============================] - 25s 15ms/step - loss: 0.2442 - sparse_categorical_accuracy: 0.9120 - val_loss: 0.3091 - val_sparse_categorical_accuracy: 0.8912\n",
      "Epoch 26/30\n",
      "1719/1719 [==============================] - 32s 19ms/step - loss: 0.2405 - sparse_categorical_accuracy: 0.9135 - val_loss: 0.2937 - val_sparse_categorical_accuracy: 0.8960\n",
      "Epoch 27/30\n",
      "1719/1719 [==============================] - 21s 12ms/step - loss: 0.2366 - sparse_categorical_accuracy: 0.9139 - val_loss: 0.2982 - val_sparse_categorical_accuracy: 0.8916\n",
      "Epoch 28/30\n",
      "1719/1719 [==============================] - 38s 22ms/step - loss: 0.2335 - sparse_categorical_accuracy: 0.9154 - val_loss: 0.2988 - val_sparse_categorical_accuracy: 0.8904\n",
      "Epoch 29/30\n",
      "1719/1719 [==============================] - 30s 17ms/step - loss: 0.2210 - sparse_categorical_accuracy: 0.9204 - val_loss: 0.3069 - val_sparse_categorical_accuracy: 0.8916\n",
      "Epoch 30/30\n",
      "1719/1719 [==============================] - 41s 24ms/step - loss: 0.2216 - sparse_categorical_accuracy: 0.9190 - val_loss: 0.2929 - val_sparse_categorical_accuracy: 0.8980\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, y_train, epochs=30,\n",
    "                    validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 4s 14ms/step - loss: 0.3274 - sparse_categorical_accuracy: 0.8868\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.32736873626708984, 0.8867999911308289]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.  , 0.  , 0.  , 0.  , 0.  , 0.01, 0.  , 0.02, 0.  , 0.97],\n",
       "       [0.  , 0.  , 1.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  ],\n",
       "       [0.  , 1.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  , 0.  ]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_new = X_test[:3]\n",
    "y_proba = model.predict(X_new)\n",
    "y_proba.round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nikos/anaconda3/envs/tsa_course/lib/python3.7/site-packages/tensorflow/python/keras/engine/sequential.py:450: UserWarning: `model.predict_classes()` is deprecated and will be removed after 2021-01-01. Please use instead:* `np.argmax(model.predict(x), axis=-1)`,   if your model does multi-class classification   (e.g. if it uses a `softmax` last-layer activation).* `(model.predict(x) > 0.5).astype(\"int32\")`,   if your model does binary classification   (e.g. if it uses a `sigmoid` last-layer activation).\n",
      "  warnings.warn('`model.predict_classes()` is deprecated and '\n"
     ]
    }
   ],
   "source": [
    "y_pred = model.predict_classes(X_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = np.argmax(y_proba, axis=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Ankle boot', 'Pullover', 'Trouser'], dtype='<U11')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(class_names)[y_pred]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "housing = fetch_california_housing()\n",
    "\n",
    "X_train_full, X_test, y_train_full, y_test = train_test_split(\n",
    "    housing.data, housing.target)\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "    X_train_full, y_train_full)\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train = scaler.fit_transform(X_train)\n",
    "X_valid = scaler.transform(X_valid)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = keras.models.Sequential([\n",
    "    keras.layers.Dense(30, activation=\"relu\", input_shape=X_train.shape[1:]),\n",
    "    keras.layers.Dense(1)]\n",
    ")\n",
    "model.compile(loss=\"mean_squared_error\", optimizer=\"sgd\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/20\n",
      "363/363 [==============================] - 3s 6ms/step - loss: 1.2555 - val_loss: 0.5932\n",
      "Epoch 2/20\n",
      "363/363 [==============================] - 2s 4ms/step - loss: 0.5738 - val_loss: 0.5077\n",
      "Epoch 3/20\n",
      "363/363 [==============================] - 2s 4ms/step - loss: 0.5246 - val_loss: 0.4836\n",
      "Epoch 4/20\n",
      "363/363 [==============================] - 2s 4ms/step - loss: 0.4641 - val_loss: 0.4706\n",
      "Epoch 5/20\n",
      "363/363 [==============================] - 2s 7ms/step - loss: 0.4776 - val_loss: 0.4571\n",
      "Epoch 6/20\n",
      "363/363 [==============================] - 2s 5ms/step - loss: 0.4454 - val_loss: 0.4520\n",
      "Epoch 7/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.4357 - val_loss: 0.4875\n",
      "Epoch 8/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.4534 - val_loss: 0.4388\n",
      "Epoch 9/20\n",
      "363/363 [==============================] - 2s 5ms/step - loss: 0.4271 - val_loss: 0.4380\n",
      "Epoch 10/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.4182 - val_loss: 0.4282\n",
      "Epoch 11/20\n",
      "363/363 [==============================] - 3s 7ms/step - loss: 0.5187 - val_loss: 0.4218\n",
      "Epoch 12/20\n",
      "363/363 [==============================] - 3s 8ms/step - loss: 0.4404 - val_loss: 0.4162\n",
      "Epoch 13/20\n",
      "363/363 [==============================] - 3s 7ms/step - loss: 0.3944 - val_loss: 0.4108\n",
      "Epoch 14/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3998 - val_loss: 0.4060\n",
      "Epoch 15/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3981 - val_loss: 0.4001\n",
      "Epoch 16/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3931 - val_loss: 0.3956\n",
      "Epoch 17/20\n",
      "363/363 [==============================] - 2s 7ms/step - loss: 0.4110 - val_loss: 0.3996\n",
      "Epoch 18/20\n",
      "363/363 [==============================] - 3s 7ms/step - loss: 0.3987 - val_loss: 0.3906\n",
      "Epoch 19/20\n",
      "363/363 [==============================] - 3s 9ms/step - loss: 0.3765 - val_loss: 0.3946\n",
      "Epoch 20/20\n",
      "363/363 [==============================] - 2s 6ms/step - loss: 0.3996 - val_loss: 0.3866\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, y_train, epochs=20, validation_data=(X_valid, y_valid))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "162/162 [==============================] - 1s 3ms/step - loss: 0.3920\n"
     ]
    }
   ],
   "source": [
    "mse_test = model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new = X_test[:3]\n",
    "y_pred = model.predict(X_new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functional API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_ = keras.layers.Input(shape=X_train.shape[1:])\n",
    "hidden1 = keras.layers.Dense(30, activation='relu') (input_)\n",
    "hidden2 = keras.layers.Dense(30, activation='relu') (hidden1)\n",
    "concat = keras.layers.Concatenate() ([input_, hidden2])\n",
    "output = keras.layers.Dense(1) (concat)\n",
    "model = keras.Model(inputs=[input_], outputs=[output])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Subclassing API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class WideAndDeepModel(keras.Model):\n",
    "    def __init__(self, units=30, activation=\"relu\", **kwargs):\n",
    "        super().__init__(**kwargs) # handles standard args (e.g., name)\n",
    "        self.hidden1 = keras.layers.Dense(units, activation=activation)\n",
    "        self.hidden2 = keras.layers.Dense(units, activation=activation)\n",
    "        self.main_output = keras.layers.Dense(1)\n",
    "        self.aux_output = keras.layers.Dense(1)\n",
    "\n",
    "    def call(self, inputs):\n",
    "        input_A, input_B = inputs\n",
    "        hidden1 = self.hidden1(input_B)\n",
    "        hidden2 = self.hidden2(hidden1)\n",
    "        concat = keras.layers.concatenate([input_A, hidden2])\n",
    "        main_output = self.main_output(concat)\n",
    "        aux_output = self.aux_output(hidden2)\n",
    "        return main_output, aux_output\n",
    "\n",
    "model = WideAndDeepModel()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using Callbacks\n",
    "\n",
    "## Early stopping with ModelCheckpoint callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_cb = keras.callbacks.ModelCheckpoint(\"my_keras_model.h5\",\n",
    "                                                save_best_only=True)\n",
    "history = model.fit(X_train, y_train, epochs=10,\n",
    "                    validation_data=(X_valid, y_valid),\n",
    "                    callbacks=[checkpoint_cb])\n",
    "model = keras.models.load_model(\"my_keras_model.h5\") # roll back to best model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "root_logdir = os.path.join(os.curdir, \"my_logs\")\n",
    "\n",
    "def get_run_logdir():\n",
    "    import time\n",
    "    run_id = time.strftime(\"run_%Y_%m_%d-%H_%M_%S\")\n",
    "    return os.path.join(root_logdir, run_id)\n",
    "\n",
    "run_logdir = get_run_logdir() # e.g., './my_logs/run_2019_06_07-15_15_22'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-09-05 16:03:15.813769: I tensorflow/core/profiler/lib/profiler_session.cc:136] Profiler session initializing.\n",
      "2021-09-05 16:03:15.813843: I tensorflow/core/profiler/lib/profiler_session.cc:155] Profiler session started.\n",
      "2021-09-05 16:03:15.815892: I tensorflow/core/profiler/lib/profiler_session.cc:172] Profiler session tear down.\n",
      "2021-09-05 16:03:16.209650: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:116] None of the MLIR optimization passes are enabled (registered 2)\n",
      "2021-09-05 16:03:16.216984: I tensorflow/core/platform/profile_utils/cpu_utils.cc:112] CPU Frequency: 2099940000 Hz\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "   3/1719 [..............................] - ETA: 2:39 - loss: 2.2696 - sparse_categorical_accuracy: 0.1233"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021-09-05 16:03:18.789207: I tensorflow/core/profiler/lib/profiler_session.cc:136] Profiler session initializing.\n",
      "2021-09-05 16:03:18.789294: I tensorflow/core/profiler/lib/profiler_session.cc:155] Profiler session started.\n",
      "2021-09-05 16:03:18.842055: I tensorflow/core/profiler/lib/profiler_session.cc:71] Profiler session collecting data.\n",
      "2021-09-05 16:03:18.846198: I tensorflow/core/profiler/lib/profiler_session.cc:172] Profiler session tear down.\n",
      "2021-09-05 16:03:18.864072: I tensorflow/core/profiler/rpc/client/save_profile.cc:137] Creating directory: ./my_logs/run_2021_09_05-16_01_35/train/plugins/profile/2021_09_05_16_03_18\n",
      "2021-09-05 16:03:18.865949: I tensorflow/core/profiler/rpc/client/save_profile.cc:143] Dumped gzipped tool data for trace.json.gz to ./my_logs/run_2021_09_05-16_01_35/train/plugins/profile/2021_09_05_16_03_18/nikos-Latitude-7490.trace.json.gz\n",
      "2021-09-05 16:03:18.891080: I tensorflow/core/profiler/rpc/client/save_profile.cc:137] Creating directory: ./my_logs/run_2021_09_05-16_01_35/train/plugins/profile/2021_09_05_16_03_18\n",
      "2021-09-05 16:03:18.904495: I tensorflow/core/profiler/rpc/client/save_profile.cc:143] Dumped gzipped tool data for memory_profile.json.gz to ./my_logs/run_2021_09_05-16_01_35/train/plugins/profile/2021_09_05_16_03_18/nikos-Latitude-7490.memory_profile.json.gz\n",
      "2021-09-05 16:03:18.906136: I tensorflow/core/profiler/rpc/client/capture_profile.cc:251] Creating directory: ./my_logs/run_2021_09_05-16_01_35/train/plugins/profile/2021_09_05_16_03_18Dumped tool data for xplane.pb to ./my_logs/run_2021_09_05-16_01_35/train/plugins/profile/2021_09_05_16_03_18/nikos-Latitude-7490.xplane.pb\n",
      "Dumped tool data for overview_page.pb to ./my_logs/run_2021_09_05-16_01_35/train/plugins/profile/2021_09_05_16_03_18/nikos-Latitude-7490.overview_page.pb\n",
      "Dumped tool data for input_pipeline.pb to ./my_logs/run_2021_09_05-16_01_35/train/plugins/profile/2021_09_05_16_03_18/nikos-Latitude-7490.input_pipeline.pb\n",
      "Dumped tool data for tensorflow_stats.pb to ./my_logs/run_2021_09_05-16_01_35/train/plugins/profile/2021_09_05_16_03_18/nikos-Latitude-7490.tensorflow_stats.pb\n",
      "Dumped tool data for kernel_stats.pb to ./my_logs/run_2021_09_05-16_01_35/train/plugins/profile/2021_09_05_16_03_18/nikos-Latitude-7490.kernel_stats.pb\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1719/1719 [==============================] - 52s 29ms/step - loss: 0.9879 - sparse_categorical_accuracy: 0.6881 - val_loss: 0.5208 - val_sparse_categorical_accuracy: 0.8248\n",
      "Epoch 2/30\n",
      "1719/1719 [==============================] - 23s 13ms/step - loss: 0.5103 - sparse_categorical_accuracy: 0.8241 - val_loss: 0.4663 - val_sparse_categorical_accuracy: 0.8296\n",
      "Epoch 3/30\n",
      "1719/1719 [==============================] - 16s 9ms/step - loss: 0.4572 - sparse_categorical_accuracy: 0.8410 - val_loss: 0.4218 - val_sparse_categorical_accuracy: 0.8534\n",
      "Epoch 4/30\n",
      "1719/1719 [==============================] - 25s 14ms/step - loss: 0.4323 - sparse_categorical_accuracy: 0.8478 - val_loss: 0.3918 - val_sparse_categorical_accuracy: 0.8668\n",
      "Epoch 5/30\n",
      "1719/1719 [==============================] - 28s 16ms/step - loss: 0.3955 - sparse_categorical_accuracy: 0.8604 - val_loss: 0.3788 - val_sparse_categorical_accuracy: 0.8660\n",
      "Epoch 6/30\n",
      "1719/1719 [==============================] - 35s 21ms/step - loss: 0.3767 - sparse_categorical_accuracy: 0.8656 - val_loss: 0.3764 - val_sparse_categorical_accuracy: 0.8684\n",
      "Epoch 7/30\n",
      "1719/1719 [==============================] - 33s 19ms/step - loss: 0.3730 - sparse_categorical_accuracy: 0.8688 - val_loss: 0.3583 - val_sparse_categorical_accuracy: 0.8762\n",
      "Epoch 8/30\n",
      "1719/1719 [==============================] - 22s 13ms/step - loss: 0.3517 - sparse_categorical_accuracy: 0.8750 - val_loss: 0.3640 - val_sparse_categorical_accuracy: 0.8732\n",
      "Epoch 9/30\n",
      "1719/1719 [==============================] - 32s 19ms/step - loss: 0.3451 - sparse_categorical_accuracy: 0.8792 - val_loss: 0.3530 - val_sparse_categorical_accuracy: 0.8744\n",
      "Epoch 10/30\n",
      "1719/1719 [==============================] - 26s 15ms/step - loss: 0.3353 - sparse_categorical_accuracy: 0.8798 - val_loss: 0.3534 - val_sparse_categorical_accuracy: 0.8740\n",
      "Epoch 11/30\n",
      "1719/1719 [==============================] - 22s 13ms/step - loss: 0.3300 - sparse_categorical_accuracy: 0.8822 - val_loss: 0.3378 - val_sparse_categorical_accuracy: 0.8802\n",
      "Epoch 12/30\n",
      "1719/1719 [==============================] - 18s 11ms/step - loss: 0.3216 - sparse_categorical_accuracy: 0.8840 - val_loss: 0.3460 - val_sparse_categorical_accuracy: 0.8742\n",
      "Epoch 13/30\n",
      "1719/1719 [==============================] - 30s 17ms/step - loss: 0.3131 - sparse_categorical_accuracy: 0.8881 - val_loss: 0.3274 - val_sparse_categorical_accuracy: 0.8842\n",
      "Epoch 14/30\n",
      "1719/1719 [==============================] - 25s 15ms/step - loss: 0.3008 - sparse_categorical_accuracy: 0.8916 - val_loss: 0.3413 - val_sparse_categorical_accuracy: 0.8770\n",
      "Epoch 15/30\n",
      "1719/1719 [==============================] - 21s 12ms/step - loss: 0.2979 - sparse_categorical_accuracy: 0.8930 - val_loss: 0.3354 - val_sparse_categorical_accuracy: 0.8814\n",
      "Epoch 16/30\n",
      "1719/1719 [==============================] - 18s 10ms/step - loss: 0.2901 - sparse_categorical_accuracy: 0.8956 - val_loss: 0.3339 - val_sparse_categorical_accuracy: 0.8812\n",
      "Epoch 17/30\n",
      "1719/1719 [==============================] - 23s 13ms/step - loss: 0.2895 - sparse_categorical_accuracy: 0.8960 - val_loss: 0.3338 - val_sparse_categorical_accuracy: 0.8816\n",
      "Epoch 18/30\n",
      "1719/1719 [==============================] - 26s 15ms/step - loss: 0.2806 - sparse_categorical_accuracy: 0.8995 - val_loss: 0.3136 - val_sparse_categorical_accuracy: 0.8862\n",
      "Epoch 19/30\n",
      "1719/1719 [==============================] - 24s 14ms/step - loss: 0.2782 - sparse_categorical_accuracy: 0.9000 - val_loss: 0.3082 - val_sparse_categorical_accuracy: 0.8868\n",
      "Epoch 20/30\n",
      "1719/1719 [==============================] - 35s 21ms/step - loss: 0.2740 - sparse_categorical_accuracy: 0.9021 - val_loss: 0.3135 - val_sparse_categorical_accuracy: 0.8892\n",
      "Epoch 21/30\n",
      "1719/1719 [==============================] - 23s 13ms/step - loss: 0.2733 - sparse_categorical_accuracy: 0.9011 - val_loss: 0.3036 - val_sparse_categorical_accuracy: 0.8936\n",
      "Epoch 22/30\n",
      "1719/1719 [==============================] - 27s 16ms/step - loss: 0.2579 - sparse_categorical_accuracy: 0.9074 - val_loss: 0.3082 - val_sparse_categorical_accuracy: 0.8892\n",
      "Epoch 23/30\n",
      "1719/1719 [==============================] - 30s 17ms/step - loss: 0.2532 - sparse_categorical_accuracy: 0.9093 - val_loss: 0.3050 - val_sparse_categorical_accuracy: 0.8896\n",
      "Epoch 24/30\n",
      "1719/1719 [==============================] - 29s 17ms/step - loss: 0.2506 - sparse_categorical_accuracy: 0.9092 - val_loss: 0.3004 - val_sparse_categorical_accuracy: 0.8916\n",
      "Epoch 25/30\n",
      "1719/1719 [==============================] - 22s 13ms/step - loss: 0.2463 - sparse_categorical_accuracy: 0.9115 - val_loss: 0.3274 - val_sparse_categorical_accuracy: 0.8810\n",
      "Epoch 26/30\n",
      "1719/1719 [==============================] - 18s 10ms/step - loss: 0.2429 - sparse_categorical_accuracy: 0.9119 - val_loss: 0.3040 - val_sparse_categorical_accuracy: 0.8874\n",
      "Epoch 27/30\n",
      "1719/1719 [==============================] - 25s 15ms/step - loss: 0.2404 - sparse_categorical_accuracy: 0.9136 - val_loss: 0.3089 - val_sparse_categorical_accuracy: 0.8880\n",
      "Epoch 28/30\n",
      "1719/1719 [==============================] - 23s 13ms/step - loss: 0.2320 - sparse_categorical_accuracy: 0.9180 - val_loss: 0.2917 - val_sparse_categorical_accuracy: 0.8956\n",
      "Epoch 29/30\n",
      "1719/1719 [==============================] - 15s 9ms/step - loss: 0.2291 - sparse_categorical_accuracy: 0.9174 - val_loss: 0.2907 - val_sparse_categorical_accuracy: 0.8938\n",
      "Epoch 30/30\n",
      "1719/1719 [==============================] - 24s 14ms/step - loss: 0.2274 - sparse_categorical_accuracy: 0.9187 - val_loss: 0.3082 - val_sparse_categorical_accuracy: 0.8896\n"
     ]
    }
   ],
   "source": [
    "tensorboard_cb = keras.callbacks.TensorBoard(run_logdir)\n",
    "history = model.fit(X_train, y_train, epochs=30,\n",
    "                    validation_data=(X_valid, y_valid),\n",
    "                    callbacks=[tensorboard_cb])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "      <iframe id=\"tensorboard-frame-b13a2144d074e931\" width=\"100%\" height=\"800\" frameborder=\"0\">\n",
       "      </iframe>\n",
       "      <script>\n",
       "        (function() {\n",
       "          const frame = document.getElementById(\"tensorboard-frame-b13a2144d074e931\");\n",
       "          const url = new URL(\"/\", window.location);\n",
       "          const port = 6006;\n",
       "          if (port) {\n",
       "            url.port = port;\n",
       "          }\n",
       "          frame.src = url;\n",
       "        })();\n",
       "      </script>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%load_ext tensorboard\n",
    "%tensorboard --logdir=./my_logs --port=6006"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "test_logdir = get_run_logdir()\n",
    "writer = tf.summary.create_file_writer(test_logdir)\n",
    "with writer.as_default():\n",
    "    for step in range(1, 1000 + 1):\n",
    "        tf.summary.scalar(\"my_scalar\", np.sin(step / 10), step=step)\n",
    "        data = (np.random.randn(100) + 2) * step / 100 # some random data\n",
    "        tf.summary.histogram(\"my_hist\", data, buckets=50, step=step)\n",
    "        images = np.random.rand(2, 32, 32, 3) # random 32×32 RGB images\n",
    "        tf.summary.image(\"my_images\", images * step / 1000, step=step)\n",
    "        texts = [\"The step is \" + str(step), \"Its square is \" + str(step**2)]\n",
    "        tf.summary.text(\"my_text\", texts, step=step)\n",
    "        sine_wave = tf.math.sin(tf.range(12000) / 48000 * 2 * np.pi * step)\n",
    "        audio = tf.reshape(tf.cast(sine_wave, tf.float32), [1, -1, 1])\n",
    "        tf.summary.audio(\"my_audio\", audio, sample_rate=48000, step=step)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fine-Tuning Neural Network Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(n_hidden=1, n_neurons=30, learning_rate=3e-3, input_shape=[8]):\n",
    "    model = keras.models.Sequential()\n",
    "    model.add(keras.layers.InputLayer(input_shape=input_shape))\n",
    "    for layer in range(n_hidden):\n",
    "        model.add(keras.layers.Dense(n_neurons, activation=\"relu\"))\n",
    "    model.add(keras.layers.Dense(1))\n",
    "    optimizer = keras.optimizers.SGD(lr=learning_rate)\n",
    "    model.compile(loss=\"mse\", optimizer=optimizer)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "keras_reg = keras.wrappers.scikit_learn.KerasRegressor(build_model, input_shape=[28, 28])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "1719/1719 [==============================] - 35s 19ms/step - loss: 7.0082 - val_loss: 4.2196\n",
      "Epoch 2/10\n",
      "1719/1719 [==============================] - 40s 24ms/step - loss: 4.3618 - val_loss: 4.0169\n",
      "Epoch 3/10\n",
      "1719/1719 [==============================] - 27s 16ms/step - loss: 4.1592 - val_loss: 3.9724\n",
      "Epoch 4/10\n",
      "1719/1719 [==============================] - 36s 21ms/step - loss: 4.1124 - val_loss: 3.8679\n",
      "Epoch 5/10\n",
      "1719/1719 [==============================] - 47s 27ms/step - loss: 4.0730 - val_loss: 3.8296\n",
      "Epoch 6/10\n",
      "1719/1719 [==============================] - 27s 16ms/step - loss: 3.9899 - val_loss: 3.8276\n",
      "Epoch 7/10\n",
      "1719/1719 [==============================] - 35s 20ms/step - loss: 3.9535 - val_loss: 3.7814\n",
      "Epoch 8/10\n",
      "1719/1719 [==============================] - 39s 22ms/step - loss: 3.9294 - val_loss: 3.7544\n",
      "Epoch 9/10\n",
      "1719/1719 [==============================] - 30s 17ms/step - loss: 3.9471 - val_loss: 3.7312\n",
      "Epoch 10/10\n",
      "1719/1719 [==============================] - 30s 17ms/step - loss: 3.8981 - val_loss: 3.7169\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fd7e87d2940>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keras_reg.fit(X_train, y_train, epochs=10,\n",
    "              validation_data=(X_valid, y_valid),\n",
    "              callbacks=[keras.callbacks.EarlyStopping(patience=10)])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "313/313 [==============================] - 2s 6ms/step - loss: 3.8884\n"
     ]
    }
   ],
   "source": [
    "mse_test = keras_reg.score(X_test, y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new = X_test[:3]\n",
    "y_pred = keras_reg.predict(X_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 17s 13ms/step - loss: 8.2945 - val_loss: 4.9479\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 16s 14ms/step - loss: 5.1490 - val_loss: 4.8968\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 13s 11ms/step - loss: 5.0327 - val_loss: 4.8854\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 11s 10ms/step - loss: 5.0916 - val_loss: 4.8978\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 15s 13ms/step - loss: 5.0270 - val_loss: 4.8788\n",
      "573/573 [==============================] - 2s 3ms/step - loss: 5.0710\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 20s 15ms/step - loss: 8.2603 - val_loss: 4.9836\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 13s 12ms/step - loss: 5.0512 - val_loss: 4.8969\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 18s 16ms/step - loss: 5.0567 - val_loss: 4.9003\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 20s 18ms/step - loss: 5.0453 - val_loss: 4.8824\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 12s 10ms/step - loss: 5.0108 - val_loss: 4.8836\n",
      "573/573 [==============================] - 1s 2ms/step - loss: 5.0777\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 8s 7ms/step - loss: 8.2337 - val_loss: 4.9702\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 8s 7ms/step - loss: 5.1541 - val_loss: 4.9006\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 11s 9ms/step - loss: 5.0916 - val_loss: 4.8856\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 7s 6ms/step - loss: 5.0779 - val_loss: 4.8915\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 7s 6ms/step - loss: 5.0666 - val_loss: 4.8831\n",
      "573/573 [==============================] - 1s 841us/step - loss: 5.0129\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 26s 22ms/step - loss: 10.8444 - val_loss: 4.5114\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 22s 20ms/step - loss: 4.6046 - val_loss: 4.1637\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 29s 25ms/step - loss: 4.2470 - val_loss: 3.9919\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 19s 17ms/step - loss: 4.1375 - val_loss: 3.9012\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 30s 26ms/step - loss: 4.1068 - val_loss: 3.8467\n",
      "573/573 [==============================] - 3s 6ms/step - loss: 4.0362\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 30s 25ms/step - loss: 11.3620 - val_loss: 4.5738\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 24s 21ms/step - loss: 4.6210 - val_loss: 4.2083\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 30s 27ms/step - loss: 4.3134 - val_loss: 4.0264\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 20s 17ms/step - loss: 4.1862 - val_loss: 3.9390\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 24s 21ms/step - loss: 4.1011 - val_loss: 3.8662\n",
      "573/573 [==============================] - 3s 5ms/step - loss: 4.0626\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 17s 14ms/step - loss: 11.9260 - val_loss: 4.5202\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 30s 26ms/step - loss: 4.6038 - val_loss: 4.1840\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 32s 28ms/step - loss: 4.3201 - val_loss: 4.0435\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 30s 26ms/step - loss: 4.2122 - val_loss: 3.9337\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 37s 32ms/step - loss: 4.1725 - val_loss: 3.8643\n",
      "573/573 [==============================] - 3s 6ms/step - loss: 4.0126\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 36s 30ms/step - loss: 10.7153 - val_loss: 4.6259\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 27s 24ms/step - loss: 4.7331 - val_loss: 4.3068\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 21s 18ms/step - loss: 4.4330 - val_loss: 4.1237\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 12s 10ms/step - loss: 4.2521 - val_loss: 4.0376\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 7s 6ms/step - loss: 4.1726 - val_loss: 3.9553\n",
      "573/573 [==============================] - 2s 3ms/step - loss: 4.1528\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 12s 10ms/step - loss: 10.5425 - val_loss: 4.5933\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 17s 15ms/step - loss: 4.7032 - val_loss: 4.2759\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 17s 15ms/step - loss: 4.4273 - val_loss: 4.1016\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 18s 15ms/step - loss: 4.2634 - val_loss: 4.0239\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 23s 20ms/step - loss: 4.1756 - val_loss: 3.9389\n",
      "573/573 [==============================] - 2s 3ms/step - loss: 4.1426\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 20s 17ms/step - loss: 10.9364 - val_loss: 4.6303\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 16s 14ms/step - loss: 4.6754 - val_loss: 4.2559\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 13s 11ms/step - loss: 4.3950 - val_loss: 4.0678\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 14s 12ms/step - loss: 4.1978 - val_loss: 3.9720\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 29s 25ms/step - loss: 4.1108 - val_loss: 3.9109\n",
      "573/573 [==============================] - 2s 4ms/step - loss: 4.0644\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 33s 28ms/step - loss: 9.4444 - val_loss: 4.3436\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 30s 26ms/step - loss: 4.4323 - val_loss: 4.0566\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 22s 19ms/step - loss: 4.1798 - val_loss: 3.9278\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 16s 14ms/step - loss: 4.0814 - val_loss: 3.8467\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 18s 16ms/step - loss: 4.0403 - val_loss: 3.7846\n",
      "573/573 [==============================] - 3s 5ms/step - loss: 3.9689\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 20s 17ms/step - loss: 9.0329 - val_loss: 4.2369\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 18s 16ms/step - loss: 4.3605 - val_loss: 4.0327\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 29s 25ms/step - loss: 4.1925 - val_loss: 3.9111\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 23s 20ms/step - loss: 4.0792 - val_loss: 3.8619\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 19s 16ms/step - loss: 3.9919 - val_loss: 3.7826\n",
      "573/573 [==============================] - 3s 4ms/step - loss: 3.9646\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 21s 18ms/step - loss: 9.9029 - val_loss: 4.2880\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 19s 17ms/step - loss: 4.4127 - val_loss: 3.9906\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 21s 18ms/step - loss: 4.1252 - val_loss: 3.8592\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 16s 14ms/step - loss: 4.0489 - val_loss: 3.8065\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 21s 18ms/step - loss: 4.0071 - val_loss: 3.7747\n",
      "573/573 [==============================] - 2s 4ms/step - loss: 3.9219\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 25s 21ms/step - loss: 11.9548 - val_loss: 4.4970\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 22s 19ms/step - loss: 4.5590 - val_loss: 4.1556\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 21s 18ms/step - loss: 4.3047 - val_loss: 4.0073\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 18s 16ms/step - loss: 4.1366 - val_loss: 3.9304\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 15s 13ms/step - loss: 4.0289 - val_loss: 3.8511\n",
      "573/573 [==============================] - 3s 5ms/step - loss: 4.0433\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 15s 13ms/step - loss: 11.3116 - val_loss: 4.4032\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 22s 20ms/step - loss: 4.4936 - val_loss: 4.1180\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 20s 18ms/step - loss: 4.2334 - val_loss: 3.9637\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 22s 20ms/step - loss: 4.1077 - val_loss: 3.8966\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 19s 17ms/step - loss: 4.0050 - val_loss: 3.8314\n",
      "573/573 [==============================] - 3s 5ms/step - loss: 4.0329\n",
      "Epoch 1/5\n",
      "1146/1146 [==============================] - 15s 13ms/step - loss: 10.9307 - val_loss: 4.5396\n",
      "Epoch 2/5\n",
      "1146/1146 [==============================] - 21s 18ms/step - loss: 4.6161 - val_loss: 4.1977\n",
      "Epoch 3/5\n",
      "1146/1146 [==============================] - 13s 11ms/step - loss: 4.3157 - val_loss: 4.0133\n",
      "Epoch 4/5\n",
      "1146/1146 [==============================] - 21s 18ms/step - loss: 4.1625 - val_loss: 3.9028\n",
      "Epoch 5/5\n",
      "1146/1146 [==============================] - 16s 14ms/step - loss: 4.1054 - val_loss: 3.8421\n",
      "573/573 [==============================] - 3s 5ms/step - loss: 3.9850\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "Cannot clone object <tensorflow.python.keras.wrappers.scikit_learn.KerasRegressor object at 0x7fd772cabeb8>, as the constructor either does not set or modifies parameter learning_rate",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_933979/2224776955.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     14\u001b[0m rnd_search_cv.fit(X_train, y_train, epochs=5,\n\u001b[1;32m     15\u001b[0m                   \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_valid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_valid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m                   callbacks=[keras.callbacks.EarlyStopping(patience=10)])\n\u001b[0m",
      "\u001b[0;32m~/anaconda3/envs/tsa_course/lib/python3.7/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     61\u001b[0m             \u001b[0mextra_args\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mextra_args\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0;31m# extra_args > 0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tsa_course/lib/python3.7/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[1;32m    875\u001b[0m             \u001b[0;31m# of the params are estimators as well.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    876\u001b[0m             self.best_estimator_ = clone(clone(base_estimator).set_params(\n\u001b[0;32m--> 877\u001b[0;31m                 **self.best_params_))\n\u001b[0m\u001b[1;32m    878\u001b[0m             \u001b[0mrefit_start_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    879\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0my\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tsa_course/lib/python3.7/site-packages/sklearn/utils/validation.py\u001b[0m in \u001b[0;36minner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     61\u001b[0m             \u001b[0mextra_args\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_args\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mextra_args\u001b[0m \u001b[0;34m<=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0;31m# extra_args > 0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/tsa_course/lib/python3.7/site-packages/sklearn/base.py\u001b[0m in \u001b[0;36mclone\u001b[0;34m(estimator, safe)\u001b[0m\n\u001b[1;32m     85\u001b[0m             raise RuntimeError('Cannot clone object %s, as the constructor '\n\u001b[1;32m     86\u001b[0m                                \u001b[0;34m'either does not set or modifies parameter %s'\u001b[0m \u001b[0;34m%\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m                                (estimator, name))\n\u001b[0m\u001b[1;32m     88\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mnew_object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: Cannot clone object <tensorflow.python.keras.wrappers.scikit_learn.KerasRegressor object at 0x7fd772cabeb8>, as the constructor either does not set or modifies parameter learning_rate"
     ]
    }
   ],
   "source": [
    "from scipy.stats import reciprocal\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "import numpy as np\n",
    "\n",
    "keras_reg = keras.wrappers.scikit_learn.KerasRegressor(build_model, input_shape=(28, 28))\n",
    "\n",
    "param_distribs = {\n",
    "    \"n_hidden\": [0, 1, 2, 3],\n",
    "    \"n_neurons\": np.arange(1, 100),\n",
    "    \"learning_rate\": reciprocal(3e-4, 3e-2),\n",
    "}\n",
    "\n",
    "rnd_search_cv = RandomizedSearchCV(keras_reg, param_distribs, n_iter=5, cv=3)\n",
    "rnd_search_cv.fit(X_train, y_train, epochs=5,\n",
    "                  validation_data=(X_valid, y_valid),\n",
    "                  callbacks=[keras.callbacks.EarlyStopping(patience=10)])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('ml')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12 (main, Apr  5 2022, 06:56:58) \n[GCC 7.5.0]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "976019bd7028fad645a45aaa2b51f9e68e9bcad9b4e9047d93bf7b882332665b"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}