neural_network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0988e417-1f1a-4be1-a811-39ef37811fa5",
   "metadata": {},
   "source": [
    "# Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2a6b7ff2-cef3-459f-9504-61fed119f99f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-10-04 19:23:28.674414: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "from ucimlrepo import fetch_ucirepo\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import copy\n",
    "import math\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Input\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b80db15e-fb98-40f4-849e-5e481e53a784",
   "metadata": {},
   "source": [
    "# Preparing the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0db35bcb-69de-4cba-8850-3f2ee08d89d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fetch dataset \n",
    "iris = fetch_ucirepo(id=53)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "363bf9f3-83f3-492f-833a-1927df95003d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data (as pandas dataframes) \n",
    "X = iris.data.features\n",
    "y = iris.data.targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "95f24ea3-32a8-45d7-afbf-01fc09e95151",
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "label = le.fit_transform(y['class'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "402e7697-42da-4948-93fc-9cd90addfdfc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/core/frame.py:4110: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  return super().drop(\n"
     ]
    }
   ],
   "source": [
    "y.drop('class', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "083d1dc8-ca99-41d7-b39f-a0a6a41f5fcd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/0_/wgcs9bsd09755tmsl79rvx300000gn/T/ipykernel_80311/1687195986.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  y['class'] = label\n"
     ]
    }
   ],
   "source": [
    "y['class'] = label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "aca56fe9-f2a9-480b-8ebc-4ea1bf340648",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_np = X.to_numpy()\n",
    "y_np = y.to_numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbd7ffc7-1237-4fae-a8db-a10789d8f56b",
   "metadata": {},
   "source": [
    "## Tensorflow Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "90e7bded-0e80-45bf-84ec-d22e339ecf3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_np = y_np.reshape(150, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "484af54c-6423-46fc-b339-1493e4442e4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential(\n",
    "    [\n",
    "        Input(shape=(4,)),\n",
    "        Dense(25, activation=\"relu\"),\n",
    "        Dense(15, activation=\"relu\"),\n",
    "        Dense(3, activation=\"linear\")\n",
    "    ]\n",
    ")       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "684dbe85-76f8-4a72-9795-037e04afdaaf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense_3 (Dense)             (None, 25)                125       \n",
      "                                                                 \n",
      " dense_4 (Dense)             (None, 15)                390       \n",
      "                                                                 \n",
      " dense_5 (Dense)             (None, 3)                 48        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 563\n",
      "Trainable params: 563\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a6ed3249-274a-4a73-ad04-f113a9006550",
   "metadata": {},
   "outputs": [],
   "source": [
    "[layer1, layer2, layer3] = model.layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "f5c6b368-c04a-4755-bfaa-25cb39df22ed",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W1: (4, 25), b1: (25,)\n",
      "W2: (25, 15), b2: (15,)\n",
      "W3: (15, 3), b3: (3,)\n"
     ]
    }
   ],
   "source": [
    "W1, b1 = layer1.get_weights()\n",
    "W2, b2 = layer2.get_weights()\n",
    "W3, b3 = layer3.get_weights()\n",
    "\n",
    "print(f'W1: {W1.shape}, b1: {b1.shape}')\n",
    "print(f'W2: {W2.shape}, b2: {b2.shape}')\n",
    "print(f'W3: {W3.shape}, b3: {b3.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "78208ccb-8ebf-423b-bc1c-8ba4fc400f2e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tf.Variable 'dense_5/kernel:0' shape=(15, 3) dtype=float32, numpy=\n",
      "array([[ 3.5025209e-01,  5.3345692e-01, -1.2304914e-01],\n",
      "       [-5.4766965e-01,  4.6755350e-01,  5.6372344e-02],\n",
      "       [-4.9092826e-01, -3.4757918e-01, -4.7534645e-02],\n",
      "       [-1.1339265e-01, -1.3634333e-01, -4.8860830e-01],\n",
      "       [ 8.0035686e-02,  2.4624860e-01,  2.5517941e-03],\n",
      "       [ 2.3115611e-01,  4.1840404e-01,  2.3897886e-02],\n",
      "       [ 2.6185948e-01, -7.8607827e-02, -3.5077500e-01],\n",
      "       [-2.5193772e-01, -1.4884785e-01,  2.9311270e-01],\n",
      "       [-2.7430505e-01,  5.5891216e-01,  3.5259724e-01],\n",
      "       [-3.0743918e-01,  5.1406217e-01,  3.1440884e-01],\n",
      "       [ 1.4187127e-01, -2.4572855e-01, -4.3074363e-01],\n",
      "       [ 3.7771773e-01,  3.1418735e-01,  2.3910946e-01],\n",
      "       [-1.8654138e-01,  4.6636045e-02,  3.9942116e-01],\n",
      "       [ 1.5757012e-01,  4.8735011e-01, -4.2487520e-01],\n",
      "       [-6.3694000e-02,  4.4483399e-01, -1.8817186e-04]], dtype=float32)>, <tf.Variable 'dense_5/bias:0' shape=(3,) dtype=float32, numpy=array([0., 0., 0.], dtype=float32)>]\n"
     ]
    }
   ],
   "source": [
    "print(model.layers[2].weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "733d4453-8009-475d-9786-5ac3a9ba05ec",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "5/5 [==============================] - 1s 3ms/step - loss: 0.6444\n",
      "Epoch 2/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.6227\n",
      "Epoch 3/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.6050\n",
      "Epoch 4/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.5867\n",
      "Epoch 5/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.5684\n",
      "Epoch 6/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.5517\n",
      "Epoch 7/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.5363\n",
      "Epoch 8/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.5211\n",
      "Epoch 9/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.5078\n",
      "Epoch 10/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.4946\n",
      "Epoch 11/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.4819\n",
      "Epoch 12/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.4702\n",
      "Epoch 13/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.4591\n",
      "Epoch 14/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.4485\n",
      "Epoch 15/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.4383\n",
      "Epoch 16/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.4282\n",
      "Epoch 17/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.4192\n",
      "Epoch 18/100\n",
      "5/5 [==============================] - 0s 8ms/step - loss: 0.4093\n",
      "Epoch 19/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.4010\n",
      "Epoch 20/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.3929\n",
      "Epoch 21/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.3847\n",
      "Epoch 22/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.3766\n",
      "Epoch 23/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.3681\n",
      "Epoch 24/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.3607\n",
      "Epoch 25/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.3526\n",
      "Epoch 26/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.3455\n",
      "Epoch 27/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.3384\n",
      "Epoch 28/100\n",
      "5/5 [==============================] - 0s 6ms/step - loss: 0.3308\n",
      "Epoch 29/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.3230\n",
      "Epoch 30/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.3170\n",
      "Epoch 31/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.3101\n",
      "Epoch 32/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.3028\n",
      "Epoch 33/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.2966\n",
      "Epoch 34/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.2884\n",
      "Epoch 35/100\n",
      "5/5 [==============================] - 0s 6ms/step - loss: 0.2826\n",
      "Epoch 36/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.2761\n",
      "Epoch 37/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.2714\n",
      "Epoch 38/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.2640\n",
      "Epoch 39/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.2598\n",
      "Epoch 40/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.2525\n",
      "Epoch 41/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.2472\n",
      "Epoch 42/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.2413\n",
      "Epoch 43/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.2365\n",
      "Epoch 44/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.2313\n",
      "Epoch 45/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.2270\n",
      "Epoch 46/100\n",
      "5/5 [==============================] - 0s 6ms/step - loss: 0.2212\n",
      "Epoch 47/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.2155\n",
      "Epoch 48/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.2131\n",
      "Epoch 49/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.2093\n",
      "Epoch 50/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.2033\n",
      "Epoch 51/100\n",
      "5/5 [==============================] - 0s 7ms/step - loss: 0.2026\n",
      "Epoch 52/100\n",
      "5/5 [==============================] - 0s 9ms/step - loss: 0.1960\n",
      "Epoch 53/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.1918\n",
      "Epoch 54/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.1875\n",
      "Epoch 55/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1842\n",
      "Epoch 56/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1810\n",
      "Epoch 57/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.1774\n",
      "Epoch 58/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1750\n",
      "Epoch 59/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1721\n",
      "Epoch 60/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.1689\n",
      "Epoch 61/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1669\n",
      "Epoch 62/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1639\n",
      "Epoch 63/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1597\n",
      "Epoch 64/100\n",
      "5/5 [==============================] - 0s 6ms/step - loss: 0.1576\n",
      "Epoch 65/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1554\n",
      "Epoch 66/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1538\n",
      "Epoch 67/100\n",
      "5/5 [==============================] - 0s 8ms/step - loss: 0.1513\n",
      "Epoch 68/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.1477\n",
      "Epoch 69/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1459\n",
      "Epoch 70/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.1432\n",
      "Epoch 71/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1421\n",
      "Epoch 72/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1402\n",
      "Epoch 73/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1374\n",
      "Epoch 74/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1352\n",
      "Epoch 75/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1361\n",
      "Epoch 76/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1313\n",
      "Epoch 77/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.1316\n",
      "Epoch 78/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1301\n",
      "Epoch 79/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.1267\n",
      "Epoch 80/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1262\n",
      "Epoch 81/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1258\n",
      "Epoch 82/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1239\n",
      "Epoch 83/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1218\n",
      "Epoch 84/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.1208\n",
      "Epoch 85/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1206\n",
      "Epoch 86/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1176\n",
      "Epoch 87/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1166\n",
      "Epoch 88/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1155\n",
      "Epoch 89/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1144\n",
      "Epoch 90/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.1146\n",
      "Epoch 91/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1116\n",
      "Epoch 92/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1114\n",
      "Epoch 93/100\n",
      "5/5 [==============================] - 0s 3ms/step - loss: 0.1101\n",
      "Epoch 94/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.1090\n",
      "Epoch 95/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.1068\n",
      "Epoch 96/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1081\n",
      "Epoch 97/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1079\n",
      "Epoch 98/100\n",
      "5/5 [==============================] - 0s 5ms/step - loss: 0.1119\n",
      "Epoch 99/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1122\n",
      "Epoch 100/100\n",
      "5/5 [==============================] - 0s 4ms/step - loss: 0.1088\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fc6354ea5e0>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.compile(\n",
    "    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "    optimizer=tf.keras.optimizers.Adam(0.001),\n",
    ")\n",
    "\n",
    "model.fit(\n",
    "    X_np,y_np,\n",
    "    epochs=100\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "1f20715e-6707-4f92-8646-2bf14115d480",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.1, 3.5, 1.4, 0.2]])"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_np[0].reshape(1, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "36a2bbb3-fb3e-4dd6-92ba-0eb527c3d951",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 186ms/step\n",
      " predicting second type: [[-6.902088   1.9760375  3.8679729]]\n"
     ]
    }
   ],
   "source": [
    "prediction = model.predict(X_np[149].reshape(1, 4))\n",
    "print(f\" predicting second type: {prediction}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "edfb3baa-0868-467d-98e7-123ccbf840a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "prediction_p = tf.nn.softmax(prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "3b5f3ef0-feea-4f3e-98b6-af5d1819ce24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_np[149]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "edf4a597-c810-485f-be29-3df5a1843928",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " predicting a Two. Probability vector: \n",
      "[[1.8265082e-05 1.3102156e-01 8.6896020e-01]]\n",
      "Total of predictions: 1.000\n"
     ]
    }
   ],
   "source": [
    "print(f\" predicting a Two. Probability vector: \\n{prediction_p}\")\n",
    "print(f\"Total of predictions: {np.sum(prediction_p):0.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "e9ca5ad8-0f48-4f32-857c-b262356bfce9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "np.argmax(prediction_p): 2\n"
     ]
    }
   ],
   "source": [
    "yhat = np.argmax(prediction_p)\n",
    "print(f\"np.argmax(prediction_p): {yhat}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59bccb30-0e33-471f-84ca-310ed5577bd5",
   "metadata": {},
   "source": [
    "## Evaluating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "436c8722-571e-41ed-8d6a-15a6d29e5903",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3116fcfc-af3c-4518-ac91-c00ce329ba55",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train, x_, y_train, y_ = train_test_split(X_np, y_np, test_size=0.40, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "94842ec3-2bca-4a20-95e4-d2faaf771278",
   "metadata": {},
   "outputs": [],
   "source": [
    "x_cv, x_test, y_cv, y_test = train_test_split(x_, y_, test_size=0.50, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "739c7d83-2cac-430a-bef2-3756552ffc98",
   "metadata": {},
   "outputs": [],
   "source": [
    "del x_, y_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "887ad5a2-fde7-4567-ac0d-232b708688fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the shape of the training set (input) is: (90, 4)\n",
      "the shape of the training set (target) is: (90, 1)\n",
      "\n",
      "the shape of the cross validation set (input) is: (30, 4)\n",
      "the shape of the cross validation set (target) is: (30, 1)\n",
      "\n",
      "the shape of the test set (input) is: (30, 4)\n",
      "the shape of the test set (target) is: (30, 1)\n"
     ]
    }
   ],
   "source": [
    "print(f\"the shape of the training set (input) is: {x_train.shape}\")\n",
    "print(f\"the shape of the training set (target) is: {y_train.shape}\\n\")\n",
    "print(f\"the shape of the cross validation set (input) is: {x_cv.shape}\")\n",
    "print(f\"the shape of the cross validation set (target) is: {y_cv.shape}\\n\")\n",
    "print(f\"the shape of the test set (input) is: {x_test.shape}\")\n",
    "print(f\"the shape of the test set (target) is: {y_test.shape}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "971ec205-577e-409a-abad-61346fb7d4c0",
   "metadata": {},
   "source": [
    "## Test different models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "17c11c50-312f-4119-bad1-be050840351f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_models():\n",
    "    \n",
    "    tf.random.set_seed(20)\n",
    "    \n",
    "    model_1 = Sequential(\n",
    "        [\n",
    "            Dense(25, activation = 'relu'),\n",
    "            Dense(15, activation = 'relu'),\n",
    "            Dense(3, activation = 'linear')\n",
    "        ],\n",
    "        name='model_1'\n",
    "    )\n",
    "\n",
    "    model_2 = Sequential(\n",
    "        [\n",
    "            Dense(20, activation = 'relu'),\n",
    "            Dense(12, activation = 'relu'),\n",
    "            Dense(12, activation = 'relu'),\n",
    "            Dense(20, activation = 'relu'),\n",
    "            Dense(3, activation = 'linear')\n",
    "        ],\n",
    "        name='model_2'\n",
    "    )\n",
    "\n",
    "    model_3 = Sequential(\n",
    "        [\n",
    "            Dense(32, activation = 'relu'),\n",
    "            Dense(16, activation = 'relu'),\n",
    "            Dense(8, activation = 'relu'),\n",
    "            Dense(4, activation = 'relu'),\n",
    "            Dense(12, activation = 'relu'),\n",
    "            Dense(3, activation = 'linear')\n",
    "        ],\n",
    "        name='model_3'\n",
    "    )\n",
    "    \n",
    "    model_list = [model_1, model_2, model_3]\n",
    "    \n",
    "    return model_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "fac1c088-d8db-4086-a917-11fd9f560b02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[2],\n",
       "       [0],\n",
       "       [0],\n",
       "       [0],\n",
       "       [1]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "614aec27-0597-40a2-a80a-84da162a0507",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3/3 [==============================] - 0s 3ms/step\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "2150c2f9-196d-46d6-9596-d798528aec25",
   "metadata": {},
   "outputs": [],
   "source": [
    "yhatt = np.array([[np.argmax(i)] for i in yhat[:5]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "8c90fd0a-de00-41f9-8a9f-06f9c188a87f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5, 1)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "yhatt[:5].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "2c22839c-6bbf-4657-aeec-832603f1bccb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30, 1)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_cv.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "60a0d846-9b61-4abd-ac5a-ccd0319a9024",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training model_1...\n",
      "Done!\n",
      "\n",
      "3/3 [==============================] - 0s 2ms/step\n",
      "1/1 [==============================] - 0s 21ms/step\n",
      "Training model_2...\n",
      "Done!\n",
      "\n",
      "3/3 [==============================] - 0s 2ms/step\n",
      "1/1 [==============================] - 0s 26ms/step\n",
      "Training model_3...\n",
      "Done!\n",
      "\n",
      "3/3 [==============================] - 0s 3ms/step\n",
      "1/1 [==============================] - 0s 22ms/step\n",
      "RESULTS:\n",
      "Model 1: Training MSE: 0.03, CV MSE: 0.00000000\n",
      "Model 2: Training MSE: 0.02, CV MSE: 0.00000000\n",
      "Model 3: Training MSE: 0.17, CV MSE: 0.10000000\n"
     ]
    }
   ],
   "source": [
    "# Initialize lists that will contain the errors for each model\n",
    "nn_train_mses = []\n",
    "nn_cv_mses = []\n",
    "\n",
    "# Build the models\n",
    "nn_models = build_models()\n",
    "\n",
    "# Loop over the the models\n",
    "for model in nn_models:\n",
    "    \n",
    "    # Setup the loss and optimizer\n",
    "    model.compile(\n",
    "    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "    optimizer=tf.keras.optimizers.Adam(0.001),\n",
    "    )\n",
    "\n",
    "    print(f\"Training {model.name}...\")\n",
    "    \n",
    "    # Train the model\n",
    "    model.fit(\n",
    "        x_train, y_train,\n",
    "        epochs=50,\n",
    "        verbose=0\n",
    "    )\n",
    "    \n",
    "    print(\"Done!\\n\")\n",
    "\n",
    "    \n",
    "    # Record the training MSEs\n",
    "    yhat = model.predict(x_train)\n",
    "    yhat = np.array([[np.argmax(i)] for i in yhat])\n",
    "    train_mse = mean_squared_error(y_train, yhat) / 2\n",
    "    nn_train_mses.append(train_mse)\n",
    "    \n",
    "    # Record the cross validation MSEs \n",
    "    yhat = model.predict(x_cv)\n",
    "    yhat = np.array([[np.argmax(i)] for i in yhat])\n",
    "    cv_mse = mean_squared_error(y_cv, yhat) / 2\n",
    "    nn_cv_mses.append(cv_mse)\n",
    "\n",
    "    \n",
    "# print results\n",
    "print(\"RESULTS:\")\n",
    "for model_num in range(len(nn_train_mses)):\n",
    "    print(\n",
    "        f\"Model {model_num+1}: Training MSE: {nn_train_mses[model_num]:.2f}, \" +\n",
    "        f\"CV MSE: {nn_cv_mses[model_num]:.8f}\"\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "474652a6-33d6-434e-8a94-89492f309c34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1 [==============================] - 0s 25ms/step\n",
      "Selected Model: 2\n",
      "Training MSE: 0.02\n",
      "Cross Validation MSE: 0.00\n",
      "Test MSE: 0.02\n"
     ]
    }
   ],
   "source": [
    "# Select the model with the lowest CV MSE\n",
    "model_num = 2\n",
    "\n",
    "# Compute the test MSE\n",
    "yhat = nn_models[model_num-1].predict(x_test)\n",
    "yhat = np.array([[np.argmax(i)] for i in yhat])\n",
    "test_mse = mean_squared_error(y_test, yhat) / 2\n",
    "\n",
    "print(f\"Selected Model: {model_num}\")\n",
    "print(f\"Training MSE: {nn_train_mses[model_num-1]:.2f}\")\n",
    "print(f\"Cross Validation MSE: {nn_cv_mses[model_num-1]:.2f}\")\n",
    "print(f\"Test MSE: {test_mse:.2f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "56734c65-31c2-491e-8d0a-ae43992718fc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_error = np.mean(yhat != y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "51b20f00-083f-431b-ba71-19c250f98dbd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.03333333333333333"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "38f51e03-3454-46e7-a80d-3be459964fa0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([16]), array([0]))"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.where(yhat != y_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}