logistic_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3ed73eb7-574d-4d92-8350-200c3136e56a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from ucimlrepo import fetch_ucirepo\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import copy\n",
    "import math\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2afe6999-583d-4222-a43b-553193cdee4e",
   "metadata": {},
   "source": [
    "# Preparing the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "da52962f-0fe9-4457-b994-99307cd44f1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# fetch dataset \n",
    "iris = fetch_ucirepo(id=53)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4b9155d4-5b30-471e-a1c8-8b1681b8f24c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'data': {'ids': None,\n",
       "  'features':      sepal length  sepal width  petal length  petal width\n",
       "  0             5.1          3.5           1.4          0.2\n",
       "  1             4.9          3.0           1.4          0.2\n",
       "  2             4.7          3.2           1.3          0.2\n",
       "  3             4.6          3.1           1.5          0.2\n",
       "  4             5.0          3.6           1.4          0.2\n",
       "  ..            ...          ...           ...          ...\n",
       "  145           6.7          3.0           5.2          2.3\n",
       "  146           6.3          2.5           5.0          1.9\n",
       "  147           6.5          3.0           5.2          2.0\n",
       "  148           6.2          3.4           5.4          2.3\n",
       "  149           5.9          3.0           5.1          1.8\n",
       "  \n",
       "  [150 rows x 4 columns],\n",
       "  'targets':               class\n",
       "  0       Iris-setosa\n",
       "  1       Iris-setosa\n",
       "  2       Iris-setosa\n",
       "  3       Iris-setosa\n",
       "  4       Iris-setosa\n",
       "  ..              ...\n",
       "  145  Iris-virginica\n",
       "  146  Iris-virginica\n",
       "  147  Iris-virginica\n",
       "  148  Iris-virginica\n",
       "  149  Iris-virginica\n",
       "  \n",
       "  [150 rows x 1 columns],\n",
       "  'original':      sepal length  sepal width  petal length  petal width           class\n",
       "  0             5.1          3.5           1.4          0.2     Iris-setosa\n",
       "  1             4.9          3.0           1.4          0.2     Iris-setosa\n",
       "  2             4.7          3.2           1.3          0.2     Iris-setosa\n",
       "  3             4.6          3.1           1.5          0.2     Iris-setosa\n",
       "  4             5.0          3.6           1.4          0.2     Iris-setosa\n",
       "  ..            ...          ...           ...          ...             ...\n",
       "  145           6.7          3.0           5.2          2.3  Iris-virginica\n",
       "  146           6.3          2.5           5.0          1.9  Iris-virginica\n",
       "  147           6.5          3.0           5.2          2.0  Iris-virginica\n",
       "  148           6.2          3.4           5.4          2.3  Iris-virginica\n",
       "  149           5.9          3.0           5.1          1.8  Iris-virginica\n",
       "  \n",
       "  [150 rows x 5 columns],\n",
       "  'headers': Index(['sepal length', 'sepal width', 'petal length', 'petal width', 'class'], dtype='object')},\n",
       " 'metadata': {'uci_id': 53,\n",
       "  'name': 'Iris',\n",
       "  'repository_url': 'https://archive.ics.uci.edu/dataset/53/iris',\n",
       "  'data_url': 'https://archive.ics.uci.edu/static/public/53/data.csv',\n",
       "  'abstract': 'A small classic dataset from Fisher, 1936. One of the earliest known datasets used for evaluating classification methods.\\n',\n",
       "  'area': 'Life Science',\n",
       "  'tasks': ['Classification'],\n",
       "  'characteristics': ['Tabular'],\n",
       "  'num_instances': 150,\n",
       "  'num_features': 4,\n",
       "  'feature_types': ['Real'],\n",
       "  'demographics': [],\n",
       "  'target_col': ['class'],\n",
       "  'index_col': None,\n",
       "  'has_missing_values': 'no',\n",
       "  'missing_values_symbol': None,\n",
       "  'year_of_dataset_creation': 1936,\n",
       "  'last_updated': 'Tue Sep 12 2023',\n",
       "  'dataset_doi': '10.24432/C56C76',\n",
       "  'creators': ['R. A. Fisher'],\n",
       "  'intro_paper': {'title': 'The Iris data set: In search of the source of virginica',\n",
       "   'authors': 'A. Unwin, K. Kleinman',\n",
       "   'published_in': 'Significance, 2021',\n",
       "   'year': 2021,\n",
       "   'url': 'https://www.semanticscholar.org/paper/4599862ea877863669a6a8e63a3c707a787d5d7e',\n",
       "   'doi': '1740-9713.01589'},\n",
       "  'additional_info': {'summary': 'This is one of the earliest datasets used in the literature on classification methods and widely used in statistics and machine learning.  The data set contains 3 classes of 50 instances each, where each class refers to a type of iris plant.  One class is linearly separable from the other 2; the latter are not linearly separable from each other.\\n\\nPredicted attribute: class of iris plant.\\n\\nThis is an exceedingly simple domain.\\n\\nThis data differs from the data presented in Fishers article (identified by Steve Chadwick,  spchadwick@espeedaz.net ).  The 35th sample should be: 4.9,3.1,1.5,0.2,\"Iris-setosa\" where the error is in the fourth feature. The 38th sample: 4.9,3.6,1.4,0.1,\"Iris-setosa\" where the errors are in the second and third features.  ',\n",
       "   'purpose': 'N/A',\n",
       "   'funded_by': None,\n",
       "   'instances_represent': 'Each instance is a plant',\n",
       "   'recommended_data_splits': None,\n",
       "   'sensitive_data': None,\n",
       "   'preprocessing_description': None,\n",
       "   'variable_info': None,\n",
       "   'citation': None}},\n",
       " 'variables':            name     role         type demographic  \\\n",
       " 0  sepal length  Feature   Continuous        None   \n",
       " 1   sepal width  Feature   Continuous        None   \n",
       " 2  petal length  Feature   Continuous        None   \n",
       " 3   petal width  Feature   Continuous        None   \n",
       " 4         class   Target  Categorical        None   \n",
       " \n",
       "                                          description units missing_values  \n",
       " 0                                               None    cm             no  \n",
       " 1                                               None    cm             no  \n",
       " 2                                               None    cm             no  \n",
       " 3                                               None    cm             no  \n",
       " 4  class of iris plant: Iris Setosa, Iris Versico...  None             no  }"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iris"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6623bca4-da96-417a-a6bd-4ea155d8e80b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# data (as pandas dataframes) \n",
    "X = iris.data.features\n",
    "y = iris.data.targets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6ae597e2-17b4-4ca1-86ca-922c393603a9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   sepal length  sepal width  petal length  petal width\n",
      "0           5.1          3.5           1.4          0.2\n",
      "1           4.9          3.0           1.4          0.2\n",
      "2           4.7          3.2           1.3          0.2\n",
      "3           4.6          3.1           1.5          0.2\n",
      "4           5.0          3.6           1.4          0.2\n",
      "(150, 4)\n",
      "<class 'pandas.core.frame.DataFrame'>\n"
     ]
    }
   ],
   "source": [
    "print(X[:5])\n",
    "print(X.shape)\n",
    "print(type(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c95bb74a-2fdc-4409-8b76-0f81c1b6ff7a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         class\n",
      "0  Iris-setosa\n",
      "1  Iris-setosa\n",
      "2  Iris-setosa\n",
      "3  Iris-setosa\n",
      "4  Iris-setosa\n",
      "(150, 1)\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "150\n"
     ]
    }
   ],
   "source": [
    "print(y[:5])\n",
    "print(y.shape)\n",
    "print(type(y))\n",
    "print(len(y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "074b6475-83d9-4b5f-b8e4-816ab3e7a500",
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "label = le.fit_transform(y['class'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8a6e47dc-c3d1-404c-be0f-8164709a5928",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/site-packages/pandas/core/frame.py:4110: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  return super().drop(\n"
     ]
    }
   ],
   "source": [
    "y.drop('class', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7d231a8e-144c-4e71-9bda-bbcc2e15ed70",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/0_/wgcs9bsd09755tmsl79rvx300000gn/T/ipykernel_1919/1687195986.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  y['class'] = label\n"
     ]
    }
   ],
   "source": [
    "y['class'] = label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "508f8547-0b92-4e21-8b62-b42e022ae567",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   class\n",
       "0      0\n",
       "1      0\n",
       "2      0\n",
       "3      0\n",
       "4      0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fc739e60-cf84-4a85-8647-d44f96479415",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_np = X.to_numpy()\n",
    "y_np = y.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "449954db-dcf2-406c-a418-537c8960c746",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5.1, 3.5, 1.4, 0.2],\n",
       "       [4.9, 3. , 1.4, 0.2],\n",
       "       [4.7, 3.2, 1.3, 0.2],\n",
       "       [4.6, 3.1, 1.5, 0.2],\n",
       "       [5. , 3.6, 1.4, 0.2]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_np[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "744303ea-1f98-4300-b849-5404e7825662",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0],\n",
       "       [0],\n",
       "       [0],\n",
       "       [0],\n",
       "       [0]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_np[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "def094b6-363f-48c6-bb59-7be66a955593",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_np = y_np.reshape(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b6927ae2-83ba-495f-89c6-4e4ad0f56aa7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(150,)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_np.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "998bb924-fdb3-4bdb-896e-6487af512155",
   "metadata": {},
   "source": [
    "# Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8979c0e1-057c-44cc-bbc2-52d5c72f8222",
   "metadata": {},
   "source": [
    "## Basic Approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "3b04a4ca-6a2d-4800-a110-00b85389314a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sigmoid(z): # Sigmoid function\n",
    "    return 1/(1+np.exp(-z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "75bf701f-5ab2-4abe-8104-503a8f1fd337",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigmoid(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "485d35ea-f84f-4780-9f55-9b5b8f3aa55e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cost(X, y, w, b, lambda_=1): # Cost function \n",
    "    m, n = X.shape\n",
    "    cost = 0\n",
    "\n",
    "    for i in range(m):\n",
    "        z = np.dot(X[i], w) + b\n",
    "        f_wb = sigmoid(z)\n",
    "        cost += -y[i]*np.log(f_wb) - (1-y[i])*np.log(1-f_wb)\n",
    "    total_cost = cost / m\n",
    "    reg_cost = sum(np.square(w))\n",
    "    \n",
    "    return total_cost + (lambda_/(2 * m)) * reg_cost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "86e99fa4-f20c-4ad7-91ab-be20b21d788c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost at initial w (zeros): 0.693\n"
     ]
    }
   ],
   "source": [
    "m, n = X_np.shape\n",
    "\n",
    "# Compute and display cost with w initialized to zeroes\n",
    "initial_w = np.zeros(n)\n",
    "initial_b = 0.\n",
    "lambda_ = 0.5\n",
    "cost = compute_cost(X_np, y_np, initial_w, initial_b, lambda_)\n",
    "print(f'Cost at initial w (zeros): {cost:.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "268da0de-d14c-494b-b86b-8a78d744cbbc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost at test w,b: 20.762\n"
     ]
    }
   ],
   "source": [
    "test_w = np.array([0.2, 0.2, 0.2, 0.2])\n",
    "test_b = -24.\n",
    "lambda_ = 0.5\n",
    "cost = compute_cost(X_np, y_np, test_w, test_b, lambda_ = 0.5)\n",
    "\n",
    "print(f'Cost at test w,b: {cost:.3f}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "44d78fd5-0b3a-48aa-9430-c60344a2bcb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_gradient(X, y, w, b, lambda_=1):\n",
    "    m, n = X.shape\n",
    "    dj_dw = np.zeros(w.shape)\n",
    "    dj_db = 0.\n",
    "\n",
    "    for i in range(m):\n",
    "        f_wb_i = sigmoid(np.dot(X[i], w) + b)\n",
    "        err_i = f_wb_i - y[i]\n",
    "        for j in range(n):\n",
    "            dj_dw[j] += err_i * X[i, j] + ((lambda_/m)) * w[j]\n",
    "        dj_db += err_i\n",
    "    dj_dw /= m\n",
    "    dj_db /= m\n",
    "\n",
    "    return dj_db, dj_dw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "d6e050f4-74ef-4d0e-b4df-fbb48238e7f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dj_db at initial w (zeros):-0.5\n",
      "dj_dw at initial w (zeros):[-3.4490000000000007, -1.3790000000000002, -3.241999999999999, -1.1933333333333325]\n"
     ]
    }
   ],
   "source": [
    "# Compute and display gradient with w initialized to zeroes\n",
    "initial_w = np.zeros(n)\n",
    "initial_b = 0.\n",
    "lambda_ = 0.5\n",
    "\n",
    "dj_db, dj_dw = compute_gradient(X_np, y_np, initial_w, initial_b, lambda_ = 0.5)\n",
    "print(f'dj_db at initial w (zeros):{dj_db}' )\n",
    "print(f'dj_dw at initial w (zeros):{dj_dw.tolist()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "b406565c-412a-411d-9c71-84f40e741f1d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dj_db at test_w: -0.999999999943004\n",
      "dj_dw at test_w: [-6.369333332982542, -2.9093333331662774, -5.119333333078041, -1.7953333332499226]\n"
     ]
    }
   ],
   "source": [
    "test_w = np.array([0.2, -0.5, 0.3, -0.4])\n",
    "test_b = -24\n",
    "lambda_ = 0.5\n",
    "dj_db, dj_dw  = compute_gradient(X_np, y_np, test_w, test_b)\n",
    "\n",
    "print('dj_db at test_w:', dj_db)\n",
    "print('dj_dw at test_w:', dj_dw.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "2708dd97-2805-4c65-8bfb-d3bff94ce979",
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient_descent(X, y, w_in, b_in, cost_function, gradient_function, alpha, num_iters, lambda_):\n",
    "    # number of training examples\n",
    "    m = len(X)\n",
    "    \n",
    "    # An array to store cost J and w's at each iteration primarily for graphing later\n",
    "    J_history = []\n",
    "    w_history = []\n",
    "    \n",
    "    for i in range(num_iters):\n",
    "\n",
    "        # Calculate the gradient and update the parameters\n",
    "        dj_db, dj_dw = gradient_function(X, y, w_in, b_in, lambda_)   \n",
    "\n",
    "        # Update Parameters using w, b, alpha and gradient\n",
    "        w_in = w_in - alpha * dj_dw               \n",
    "        b_in = b_in - alpha * dj_db              \n",
    "       \n",
    "        # Save cost J at each iteration\n",
    "        if i<100000:      # prevent resource exhaustion \n",
    "            cost =  cost_function(X, y, w_in, b_in, lambda_)\n",
    "            J_history.append(cost)\n",
    "\n",
    "        # Print cost every at intervals 10 times or as many iterations if < 10\n",
    "        if i% math.ceil(num_iters/10) == 0 or i == (num_iters-1):\n",
    "            w_history.append(w_in)\n",
    "            print(f\"Iteration {i:4}: Cost {float(J_history[-1]):8.2f}   \")\n",
    "        \n",
    "    return w_in, b_in, J_history, w_history #return w and J,w history for graphing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "a65e559e-91cc-4a78-9d3d-502c45f5d8a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration    0: Cost     7.92   \n",
      "Iteration  300: Cost    -1.75   \n",
      "Iteration  600: Cost    -2.75   \n",
      "Iteration  900: Cost    -3.55   \n",
      "Iteration 1200: Cost    -4.31   \n",
      "Iteration 1500: Cost    -5.07   \n",
      "Iteration 1800: Cost    -5.82   \n",
      "Iteration 2100: Cost    -6.58   \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/0_/wgcs9bsd09755tmsl79rvx300000gn/T/ipykernel_39213/2969695601.py:8: RuntimeWarning: divide by zero encountered in log\n",
      "  cost += -y[i]*np.log(f_wb) - (1-y[i])*np.log(1-f_wb)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 2400: Cost     -inf   \n",
      "Iteration 2700: Cost     -inf   \n",
      "Iteration 2999: Cost     -inf   \n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "intial_w = 0.01 * (np.random.rand(4).reshape(-1,1) - 0.5)\n",
    "initial_b = -8\n",
    "\n",
    "\n",
    "# Some gradient descent settings\n",
    "iterations = 3000\n",
    "alpha = 0.001\n",
    "\n",
    "w,b, J_history,_ = gradient_descent(X_np ,y_np, initial_w, initial_b, \n",
    "                                   compute_cost, compute_gradient, alpha, iterations, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "aef2a1b7-97eb-4c1b-9a08-e7b73cfc3156",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(X, w, b):\n",
    "    m = X.shape[0]\n",
    "    p = np.zeros(m)\n",
    "\n",
    "    for i in range(m):\n",
    "        z = np.dot(w, X[i]) + b\n",
    "        f_wb = sigmoid(z)\n",
    "        if f_wb >= 0.5:\n",
    "            p[i] = 1\n",
    "        elif f_wb < 0.5:\n",
    "            p[i] = 0\n",
    "        else:\n",
    "            p[i] = 2\n",
    "\n",
    "    return p\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "f7418765-4f63-4bca-97db-f69b0286005f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Output of predict: shape (4,), value [1. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "tmp_w = np.random.randn(4)\n",
    "tmp_b = 0.3    \n",
    "tmp_X = np.random.randn(4, 4) - 0.5\n",
    "\n",
    "tmp_p = predict(tmp_X, tmp_w, tmp_b)\n",
    "print(f'Output of predict: shape {tmp_p.shape}, value {tmp_p}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "14dde4bf-467a-47f5-bad8-17c053ca6de9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy: 33.333333\n"
     ]
    }
   ],
   "source": [
    "p = predict(X_np, w,b)\n",
    "print('Train Accuracy: %f'%(np.mean(p == y_np) * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "926578dd-a786-40c7-84d6-551d8b8f87fb",
   "metadata": {},
   "source": [
    "## Scikit approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "219c1c06-da1f-4abe-bb9c-a2cdc3856b00",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "9b1abb19-f7ce-4824-b355-d4d8c69815f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = LogisticRegression(random_state=0).fit(X_np, y_np)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "0b3e34da-759f-4aff-b136-2c0f8b68f996",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1,\n",
       "       1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "       2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2])"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.predict(X_np)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "300665b1-7924-469f-b38a-8dd05567f90e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy: 0.9733333333333334\n"
     ]
    }
   ],
   "source": [
    "print(f'Train Accuracy: {clf.score(X_np, y_np)}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}