FINAL_MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib.pyplot import figure\n",
    "from sklearn.metrics import f1_score\n",
    "from scipy.sparse import csr_matrix\n",
    "from numpy import count_nonzero\n",
    "import matplotlib.pyplot as plt\n",
    "from numpy import linalg as LA\n",
    "import collections, numpy\n",
    "from numpy import array\n",
    "from time import time\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.io\n",
    "import datetime\n",
    "import random\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "mushrooms = scipy.io.loadmat('C:\\\\Users\\\\Hunor\\\\Documents\\\\Moj Master rad\\\\Podaci za testiranje NKNKJTO\\\\MNIST.mat')\n",
    "train = pd.DataFrame(mushrooms['Xtrain'])\n",
    "\n",
    "test = pd.DataFrame(mushrooms['Xtest'])\n",
    "\n",
    "#labels = pd.DataFrame(mushrooms['ztrain'].toarray())\n",
    "labels = pd.DataFrame(mushrooms['ztrain'])\n",
    "labels = labels[0]\n",
    "#labels = np.array(labels[0])\n",
    "\n",
    "#labels_test = pd.DataFrame(mushrooms['ztest'].toarray())\n",
    "labels_test = pd.DataFrame(mushrooms['ztest'])\n",
    "labels_test = labels_test[0]\n",
    "#labels_test = np.array(labels_test[0])\n",
    "\n",
    "x0 = np.random.uniform(0,1,train.shape[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def aimh(N_k):\n",
    "    return (N_max-N_k)/N_max"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hinge_loss(x,N):\n",
    "    prediction = train.iloc[:N,:].dot(x)\n",
    "    maximums = np.maximum(0,1-labels[:N]*prediction)\n",
    "    return l/2*LA.norm(x)**2+1/N*maximums.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hl_der(x,N):\n",
    "    y=train.iloc[:N,:].dot(x)*labels[:N]\n",
    "    maximumi= 1-y\n",
    "    izvodi_svi = (train.iloc[:N,:].T*labels[:N]).T\n",
    "    izvodi_veci = izvodi_svi[maximumi>0]\n",
    "    izvodi_nula = izvodi_svi[maximumi==0]\n",
    "    beta=np.random.uniform(0,1,np.size(izvodi_nula))\n",
    "    if len(izvodi_nula)!=0:\n",
    "        return np.array(l*x-izvodi_veci.sum()/N-np.dot(izvodi_nula,beta)/N)\n",
    "    elif len(izvodi_nula)==0:\n",
    "        return np.array(l*x-(izvodi_veci.sum())/N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def new_sample(N_k): #working\n",
    "    return math.ceil(N_max*(1-r*aimh(N_k)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merit_function(x_k, N_k, theta_k):\n",
    "    return theta_k*hinge_loss(x_k, N_k)+(1-theta_k)*aimh(N_k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_theta(x, N_tilda, N, theta):\n",
    "    sum1 = merit_function(x, N_tilda, theta)-merit_function(x, N, theta)\n",
    "    sum2 = (1-r)/2*(aimh(N_tilda)-aimh(N))\n",
    "    if sum1 <= sum2:\n",
    "        #print(\"Theta - Same\")\n",
    "        return theta\n",
    "    else:\n",
    "        sum3 = (1+r)*(aimh(N)-aimh(N_tilda))\n",
    "        sum4 = hinge_loss(x, N_tilda)-hinge_loss(x, N)+aimh(N)-aimh(N_tilda)\n",
    "        #print(\"Theta - Upgrade\")\n",
    "        return sum3/(2*sum4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_Y(x, p, N, B):\n",
    "    return np.matmul(np.matmul(p,np.linalg.inv(B)),p)/2 + sup_gp(x, p, N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sup_gp(x, p, N):\n",
    "    sum1 = 0\n",
    "    sum2 = 0\n",
    "    for i in range(N):\n",
    "        check = 1-labels[i]*np.matmul(train.iloc[i,:],x)\n",
    "        if check == 0:\n",
    "            inf = labels[i]*np.matmul(train.iloc[i,:],p)\n",
    "            if inf < 0:\n",
    "                sum1 += inf\n",
    "        if check > 0:\n",
    "            sum2 += labels[i]*train.iloc[i,:]\n",
    "    return np.matmul(l*x - (sum2/N),p) - sum1/N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def argsup_gp(x, p, N): #working\n",
    "    sum1 = np.zeros(x.shape)\n",
    "    sum2 = np.zeros(x.shape)\n",
    "    for i in range(N):\n",
    "        check = 1-labels[i]*np.matmul(train.iloc[i,:],x) \n",
    "        if check > 0:\n",
    "            sum1 += labels[i]*train.iloc[i][:]\n",
    "        if check == 0:\n",
    "            if labels[i]*np.matul(train.iloc[i,:],p) < 0:\n",
    "                sum2 += lables[i]*train.iloc[i][:]           \n",
    "    return np.array(l*x-sum1/N-sum2/N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def descentDirection(g0_tilda, e, i_max, B, x, N): #working\n",
    "    G_tilda = []\n",
    "    G_bar = []\n",
    "    P = []\n",
    "    E = []\n",
    "    \n",
    "    #Step 0\n",
    "    g0_bar = g0_tilda\n",
    "    G_bar.append(g0_bar)\n",
    "    G_tilda.append(g0_tilda)\n",
    "    P.append(np.matmul(-B, G_tilda[0]))\n",
    "    \n",
    "    #Step 1\n",
    "    G_tilda.append(argsup_gp(x, P[0], N))\n",
    "        \n",
    "    #Step 2\n",
    "    E.append(np.matmul(P[0],G_tilda[1])-np.matmul(P[0],G_bar[0]))\n",
    "    \n",
    "    #Step 3\n",
    "    i = 0\n",
    "    while (np.matmul(G_tilda[i+1],P[i])>0 or E[0]>e) and E[i]>0 and i<i_max:\n",
    "        first = np.matmul(G_bar[i]-G_tilda[i+1],np.matmul(B,G_bar[i]))\n",
    "        second = np.matmul(G_bar[i]-G_tilda[i+1],np.matmul(B,G_bar[i]-G_tilda[i+1]))\n",
    "        ni = min(1,first/second)\n",
    "        G_bar.append((1-ni)*G_bar[i]+ni*G_tilda[i+1])\n",
    "        P.append((1-ni)*P[i]-ni*np.matmul(B,G_tilda[i+1]))\n",
    "        G_tilda.append(argsup_gp(x, P[i+1], N))\n",
    "        e_list = []\n",
    "        j = 0\n",
    "        while j <= i+1:\n",
    "            sum1 = np.matmul(P[j],G_tilda[j+1])\n",
    "            sum2 = np.matmul(P[j],G_bar[j])+np.matmul(P[i+1],G_bar[i+1])\n",
    "            e_list.append(sum1-sum2/2)\n",
    "            j += 1\n",
    "        E.append(min(e_list))\n",
    "        i += 1\n",
    "        \n",
    "    #Step 4\n",
    "    list_p = []\n",
    "    j = 0\n",
    "    while j <= i:\n",
    "        list_p.append(model_Y(x, P[j], N, B))\n",
    "        j += 1\n",
    "    p_final = P[np.argmin(list_p)]\n",
    "    \n",
    "    #STEP5\n",
    "    if sup_gp(x, p_final, N) < 0:\n",
    "        #print(\"Iteration in finding P is %s\" %i)\n",
    "        return p_final\n",
    "    else:\n",
    "        print(\"!!! FAILED TO FIND DESCENT DIRECTION!!!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check1(x, alpha, p, N_trial, N_tilda):\n",
    "    sum1 = hinge_loss(x+alpha*p, N_trial)\n",
    "    sum2 = hinge_loss(x, N_tilda)\n",
    "    sum3 = -gama*alpha*(LA.norm(p,2)**2)\n",
    "    return (sum1-sum2)<=sum3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check2(N_trial, N_tilda, alpha, p):\n",
    "    sum1 = aimh(N_trial)\n",
    "    sum2 = aimh(N_tilda)\n",
    "    sum3 = gama_k*alpha*alpha*(LA.norm(p,2)**2)\n",
    "    return sum1<=(sum2+sum3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check3(x, alpha, p, N_trial, theta, N_start, N_tilda):\n",
    "    sum1 = merit_function(x+alpha*p, N_trial, theta)\n",
    "    sum2 = merit_function(x, N_start, theta)\n",
    "    sum3 = (1-r)*(aimh(N_tilda)-aimh(N_start))/2\n",
    "    return (sum1-sum2)<=sum3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def BFGS(B, g_new, g_old, x_new, x_old):\n",
    "    I = np.identity(g_new.size)\n",
    "    y = g_new-g_old\n",
    "    s = x_new-x_old\n",
    "    sT = np.reshape(s,(1,-1)) # Dimension: 1xN\n",
    "    s = np.reshape(s,(-1,1)) # Dimension: Nx1\n",
    "    yT = np.reshape(y,(1,-1))\n",
    "    y = np.reshape(y,(-1,1))\n",
    "    ro = 1/(yT@s)\n",
    "    sum1 = I-ro*(s@yT)\n",
    "    sum2 = I-ro*(y@sT)\n",
    "    sum3 = ro*(s@sT)\n",
    "    return (np.matmul(np.matmul(sum1,B),sum2)+sum3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def theta_k(theta):\n",
    "    return N_max*theta/(1-theta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def p_N_trial(N_start, N_tilda, theta, theta_k, alpha, p ,x):\n",
    "    sum1 = N_start + ((1-r)/2)*(N_tilda-N_start)/(1-theta)\n",
    "    sum2 = theta_k*(gama*alpha*(LA.norm(p,2)**2) - hinge_loss(x, N_tilda) + hinge_loss(x, N_start))\n",
    "    return round(math.floor(sum1-sum2))\n",
    "    #return sum1, sum2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def first_N_trial(N_start, N_tilda, theta, theta_k, alpha, x):\n",
    "    sum1 = N_start+(1-r)/2*(N_tilda-N_start)/(1-theta)\n",
    "    sum2 = theta_k*(gama*alpha-hinge_loss(x, N_tilda)+hinge_loss(x, N_start))\n",
    "    return math.ceil(sum1-sum2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def candidates(N_trial, N_tilda):\n",
    "    storage = []\n",
    "    storage.append(int(N_trial))\n",
    "    storage.append(int(math.ceil((N_trial+N_tilda)/2)))\n",
    "    storage.append(N_tilda)\n",
    "    return storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scores(k, x):\n",
    "    true_pos = 0\n",
    "    true_neg = 0\n",
    "    false_pos = 0\n",
    "    false_neg = 0\n",
    "    correct = 0\n",
    "    for i in range(test.shape[0]):\n",
    "        value = np.matmul(test.iloc[i,:], x)\n",
    "        if value > 0 and labels_test[i] > 0:\n",
    "            true_pos += 1\n",
    "        if value < 0 and labels_test[i] < 0:\n",
    "            true_neg += + 1\n",
    "        if value < 0 and labels_test[i] > 0:\n",
    "            false_pos += + 1\n",
    "        if value > 0 and labels_test[i] < 0:\n",
    "            false_neg += + 1\n",
    "        if value*labels_test[i]>0:\n",
    "            correct += 1      \n",
    "    F1 = round(true_pos/(true_pos + (false_pos+false_neg)/2),3)    \n",
    "    Accuracy = round(correct / test.shape[0], 3)\n",
    "    print(\"Iteration: %s | Accuracy: %s | F1: %s\" %(k, F1, Accuracy))\n",
    "    return F1, Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backtracking_IRNS(B, x, N_trial, N_tilda, N_start, theta):\n",
    "    alpha_start = 0.5\n",
    "    degree = 0\n",
    "    g0 = hl_der(x, N_trial[0])\n",
    "    g1 = hl_der(x, N_trial[1])\n",
    "    g2 = hl_der(x, N_trial[2])\n",
    "    \n",
    "    p0 = descentDirection(g0, e, i_max, B, x, N_trial[0])\n",
    "    #print(\"P0 done\")\n",
    "    p1 = descentDirection(g1, e, i_max, B, x, N_trial[1])\n",
    "    #print(\"P1 done\")\n",
    "    p2 = descentDirection(g2, e, i_max, B, x, N_trial[2])\n",
    "    #print(\"P2 done\")\n",
    "    \n",
    "    while True:\n",
    "        alpha = alpha_start**degree\n",
    "        \n",
    "        first = check1(x, alpha, p0, N_trial[0], N_tilda)\n",
    "        second = check2(N_trial[0], N_tilda, alpha, p0)\n",
    "        third = check3(x, alpha, p0, N_trial[0], theta, N_start, N_tilda)\n",
    "        if first and second and third:\n",
    "            print(\"Alpha: %s | Trial value: %s\" %(alpha,0))\n",
    "            return alpha, p0, N_trial[0]\n",
    "            break\n",
    "        else:\n",
    "            first = check1(x, alpha, p1, N_trial[1], N_tilda)\n",
    "            second = check2(N_trial[1], N_tilda, alpha, p1) \n",
    "            third = check3(x, alpha, p1, N_trial[1], theta, N_start, N_tilda)\n",
    "            if first and second and third:\n",
    "                print(\"Alpha: %s | Trial value: %s\" %(alpha,1))\n",
    "                return alpha, p1, N_trial[1]\n",
    "                break\n",
    "            else:\n",
    "                first = check1(x, alpha, p2, N_trial[2], N_tilda)\n",
    "                second = check2(N_trial[2], N_tilda, alpha, p2) \n",
    "                third = check3(x, alpha, p2, N_trial[2], theta, N_start, N_tilda)\n",
    "                if first and second and third:\n",
    "                    print(\"Alpha: %s | Trial value: %s\" %(alpha,2))\n",
    "                    return alpha, p2, N_trial[2]\n",
    "                    break\n",
    "                else:\n",
    "                    if degree >= 10:\n",
    "                        print(\"forced to stop\")\n",
    "                        return alpha, p2, N_trial[2]\n",
    "                        break\n",
    "                    degree += 1\n",
    "                    print(degree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def backtracking_normal(B, x, N_trial, N_tilda, N_start, theta):\n",
    "    alpha_start = 0.5\n",
    "    degree = 0\n",
    "    g = hl_der(x, N_trial)\n",
    "    p = descentDirection(g,e,i_max,B,x,N_trial)\n",
    "    while True:\n",
    "        alpha = alpha_start**degree\n",
    "        \n",
    "        first = check1(x, alpha, p, N_trial, N_tilda)\n",
    "        third = check3(x, alpha, p, N_trial, theta, N_start, N_tilda)\n",
    "        #print(first, third)\n",
    "        if first and third:\n",
    "            #print(\"Alpha: %s\" %alpha)\n",
    "            return alpha, p\n",
    "            break\n",
    "        else:\n",
    "            if degree >= 10:\n",
    "                #print(\"forced to stop\")\n",
    "                return alpha, p\n",
    "                break\n",
    "            degree += 1\n",
    "            #print(degree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_max = train.shape[0]\n",
    "B0 = np.identity(x0.size)\n",
    "gama = math.pow(10,-4)\n",
    "e = math.pow(10,-4)\n",
    "m = math.pow(10,-4)\n",
    "theta0 = 0.9\n",
    "i_max = 100\n",
    "gama_k = 1\n",
    "r = 0.95\n",
    "l = math.pow(10,-5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def algorithm(method): \n",
    "    start = time()\n",
    "    \n",
    "    max_fev = 0\n",
    "    MAX_FEV = []\n",
    "    MAX_FEV.append(max_fev)\n",
    "    \n",
    "    f_Ntrain = []\n",
    "    N_tilda = []\n",
    "    trial_N = []\n",
    "    ITER = []\n",
    "    ACC = []\n",
    "    F1 = []\n",
    "    T = []\n",
    "    X = []\n",
    "    P = []\n",
    "    B = []\n",
    "    G = []\n",
    "    C = []\n",
    "    A = []\n",
    "    N = []\n",
    "    \n",
    "    X.append(x0)\n",
    "    if method == \"IRNS\":\n",
    "        N0 = math.ceil(0.1*N_max)\n",
    "        N.append(N0)\n",
    "        g0 = hl_der(x0, N[0])\n",
    "        G.append(g0)\n",
    "        max_fev += N[0]\n",
    "        MAX_FEV.append(max_fev)\n",
    "        f_Ntrain.append(hinge_loss(X[0],N_max))\n",
    "         \n",
    "    if method == \"HEUR\":\n",
    "        N0 = math.ceil(0.1*N_max)\n",
    "        N.append(N0)\n",
    "        g0 = hl_der(x0, N[0])\n",
    "        G.append(g0)\n",
    "        max_fev += N[0]\n",
    "        MAX_FEV.append(max_fev)\n",
    "        f_Ntrain.append(hinge_loss(X[0],N_max))\n",
    "        \n",
    "    if method == \"FSS\":\n",
    "        N0 = train.shape[0]\n",
    "        N.append(N0)\n",
    "        g0 = hl_der(X[0], N[0])\n",
    "        G.append(g0)\n",
    "        max_fev += N[0]\n",
    "        MAX_FEV.append(max_fev)\n",
    "        f_Ntrain.append(hinge_loss(X[0],N_max))\n",
    "        \n",
    "    N_tilda.append(0)\n",
    "    T.append(theta0)\n",
    "    B.append(B0)\n",
    "\n",
    "    \n",
    "    k = 0\n",
    "    ITER.append(0)\n",
    "    #while max_fev <= 10**5:\n",
    "    while k <= 100:\n",
    "        if method == \"IRNS\":\n",
    "            N_tilda.append(new_sample(N[k]))\n",
    "            T.append(find_theta(X[k], N_tilda[k+1], N[k], T[k]))\n",
    "            \n",
    "            #N_trial = round(0.8*N_tilda[-1])\n",
    "            if k == 0:\n",
    "                N_trial = first_N_trial(N[0], N_tilda[1], T[1], theta_k(T[1]), 1, X[0])\n",
    "            else:\n",
    "                N_trial = p_N_trial(N[k], N_tilda[k+1], T[k+1], theta_k(T[k+1]), A[-1], P[k-1] ,X[k])\n",
    "            \n",
    "            trial_N.append(N_trial)\n",
    "            three_canditates = candidates(N_trial, N_tilda[k+1])\n",
    "            C.append(three_canditates)\n",
    "            print(C[k])\n",
    "            alpha, p, N_new = backtracking_IRNS(B[k],X[k],three_canditates,N_tilda[k+1],N[k],T[k+1])\n",
    "            N.append(N_new)\n",
    "            max_fev += N[-1]\n",
    "            MAX_FEV.append(max_fev)\n",
    "            \n",
    "        if method == \"HEUR\":\n",
    "            N_tilda.append(new_sample(N[k]))\n",
    "            T.append(find_theta(X[k], N_tilda[k+1], N[k], T[k]))\n",
    "            N_trial = round(min(1.1*N[k],N_max))\n",
    "            N.append(N_trial)\n",
    "            alpha, p = backtracking_normal(B[k],X[k],N_trial,N_tilda[k+1],N[k],T[k+1])\n",
    "            max_fev += N[-1]\n",
    "            MAX_FEV.append(max_fev)\n",
    "            \n",
    "        if method == \"FSS\":\n",
    "            N_tilda.append(train.shape[0])\n",
    "            T.append(find_theta(X[k], N_tilda[k+1], N[k], T[k]))\n",
    "            N_trial = train.shape[0]\n",
    "            alpha, p = backtracking_normal(B[k], X[k], N_trial, N_tilda[k+1], N[k], T[k+1])\n",
    "            N.append(train.shape[0])\n",
    "            max_fev += N[-1]\n",
    "            MAX_FEV.append(max_fev)\n",
    "\n",
    "        P.append(p)\n",
    "        A.append(alpha)\n",
    "\n",
    "        #Step 4\n",
    "        sk = alpha*P[k]\n",
    "        X.append(X[k]+sk)\n",
    "        f_Ntrain.append(hinge_loss(X[-1],N_max))\n",
    "    \n",
    "\n",
    "        F, Accuracy = scores(k, X[-1])\n",
    "        F1.append(F)\n",
    "        ACC.append(Accuracy)\n",
    "\n",
    "        #print(\"Descent: %s | Semiposdef: %s \" %(np.matmul(G[-1],P[-1])<0,np.all(np.linalg.eigvals(B[-1]) > 0)))\n",
    "\n",
    "        #Step 5\n",
    "        G.append(hl_der(X[k+1], N[k+1]))\n",
    "        yk = G[k+1]-G[k]\n",
    "        if np.matmul(sk,yk) >= m*LA.norm(yk,2)**2:\n",
    "            B.append(BFGS(B[k], G[k+1], G[k], X[k+1], X[k]))\n",
    "        else:\n",
    "            B.append(B[k])\n",
    "\n",
    "        #Step 6\n",
    "        k += 1 \n",
    "        ITER.append(k)\n",
    "        print(\"-------------------------------------\")\n",
    "    end = time()\n",
    "    b = end - start\n",
    "    print(' ')\n",
    "    print('Time elapsed: %s minutes and %s seconds.'%(np.floor(b / 60), np.round(b % 60)))\n",
    "    return F1, ACC, MAX_FEV, f_Ntrain, N, ITER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-2708008, -1349653, 8701]\n",
      "!!! FAILED TO FIND DESCENT DIRECTION!!!\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for *: 'float' and 'NoneType'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-35-ecda6b72525b>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mF1_IRNS\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mACC_IRNS\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mMAX_FEV_IRNS\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf_Ntrain_IRNS\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_IRNS\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mITER_IRNS\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0malgorithm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"IRNS\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-34-f691d140a40e>\u001b[0m in \u001b[0;36malgorithm\u001b[1;34m(method)\u001b[0m\n\u001b[0;32m     72\u001b[0m             \u001b[0mC\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mthree_canditates\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     73\u001b[0m             \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mC\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 74\u001b[1;33m             \u001b[0malpha\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_new\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbacktracking_IRNS\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mB\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mthree_canditates\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mN_tilda\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mN\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mT\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     75\u001b[0m             \u001b[0mN\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mN_new\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     76\u001b[0m             \u001b[0mmax_fev\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mN\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-31-3b15d3c314f0>\u001b[0m in \u001b[0;36mbacktracking_IRNS\u001b[1;34m(B, x, N_trial, N_tilda, N_start, theta)\u001b[0m\n\u001b[0;32m     32\u001b[0m                 \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     33\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 34\u001b[1;33m                 \u001b[0mfirst\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mp2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_trial\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_tilda\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     35\u001b[0m                 \u001b[0msecond\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck2\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mN_trial\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_tilda\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mp2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m                 \u001b[0mthird\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck3\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mp2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_trial\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtheta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_start\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_tilda\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-22-befc512d288b>\u001b[0m in \u001b[0;36mcheck1\u001b[1;34m(x, alpha, p, N_trial, N_tilda)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mcheck1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_trial\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_tilda\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[0msum1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhinge_loss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0malpha\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_trial\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m     \u001b[0msum2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhinge_loss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_tilda\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0msum3\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m-\u001b[0m\u001b[0mgama\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0malpha\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mLA\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m**\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0msum1\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0msum2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m<=\u001b[0m\u001b[0msum3\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: unsupported operand type(s) for *: 'float' and 'NoneType'"
     ]
    }
   ],
   "source": [
    "F1_IRNS, ACC_IRNS, MAX_FEV_IRNS, f_Ntrain_IRNS, N_IRNS, ITER_IRNS = algorithm(\"IRNS\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 0 | Accuracy: 0.007 | F1: 0.493\n",
      "-------------------------------------\n",
      "!!! FAILED TO FIND DESCENT DIRECTION!!!\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for *: 'float' and 'NoneType'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-36-7dd4ef4c7931>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mF1_HEUR\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mACC_HEUR\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mMAX_FEV_HEUR\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf_Ntrain_HEUR\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_HEUR\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mITER_HEUR\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0malgorithm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"HEUR\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-34-f691d140a40e>\u001b[0m in \u001b[0;36malgorithm\u001b[1;34m(method)\u001b[0m\n\u001b[0;32m     82\u001b[0m             \u001b[0mN_trial\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mround\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1.1\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mN\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mN_max\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     83\u001b[0m             \u001b[0mN\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mN_trial\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 84\u001b[1;33m             \u001b[0malpha\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbacktracking_normal\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mB\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mN_trial\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mN_tilda\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mN\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mT\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m+\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     85\u001b[0m             \u001b[0mmax_fev\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mN\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     86\u001b[0m             \u001b[0mMAX_FEV\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmax_fev\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-32-3d7e884afbc7>\u001b[0m in \u001b[0;36mbacktracking_normal\u001b[1;34m(B, x, N_trial, N_tilda, N_start, theta)\u001b[0m\n\u001b[0;32m      7\u001b[0m         \u001b[0malpha\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0malpha_start\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0mdegree\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      8\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 9\u001b[1;33m         \u001b[0mfirst\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_trial\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_tilda\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     10\u001b[0m         \u001b[0mthird\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck3\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_trial\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtheta\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_start\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_tilda\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m         \u001b[1;31m#print(first, third)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-22-befc512d288b>\u001b[0m in \u001b[0;36mcheck1\u001b[1;34m(x, alpha, p, N_trial, N_tilda)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mcheck1\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_trial\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_tilda\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[0msum1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhinge_loss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0malpha\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_trial\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m     \u001b[0msum2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mhinge_loss\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mN_tilda\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0msum3\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m-\u001b[0m\u001b[0mgama\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0malpha\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mLA\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnorm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mp\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m**\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0msum1\u001b[0m\u001b[1;33m-\u001b[0m\u001b[0msum2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m<=\u001b[0m\u001b[0msum3\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mTypeError\u001b[0m: unsupported operand type(s) for *: 'float' and 'NoneType'"
     ]
    }
   ],
   "source": [
    "F1_HEUR, ACC_HEUR, MAX_FEV_HEUR, f_Ntrain_HEUR, N_HEUR, ITER_HEUR = algorithm(\"HEUR\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "F1_FSS, ACC_FSS, MAX_FEV_FSS, f_Ntrain_FSS, N_FSS, ITER_FSS = algorithm(\"FSS\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Graphics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 5))\n",
    "plt.plot(ACC_IRNS)\n",
    "plt.plot(ACC_HEUR)\n",
    "plt.plot(ACC_FSS)\n",
    "plt.legend([\"IRNS\", \"Heuristics\",\"Full sample size\"], loc =\"lower right\")\n",
    "plt.xlabel(\"Iteration\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 5))\n",
    "plt.plot(F1_IRNS)\n",
    "plt.plot(F1_HEUR)\n",
    "plt.plot(F1_FSS)\n",
    "plt.legend([\"IRNS\", \"Heuristics\",\"Full sample size\"], loc =\"lower right\")\n",
    "plt.xlabel(\"Iteration\")\n",
    "plt.ylabel(\"F1 Score\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 5))\n",
    "plt.plot(MAX_FEV_IRNS)\n",
    "plt.plot(MAX_FEV_HEUR)\n",
    "plt.plot(MAX_FEV_FSS)\n",
    "plt.legend([\"IRNS\", \"Heuristics\",\"Full sample size\"], loc =\"upper left\")\n",
    "plt.xlabel(\"Iteration\")\n",
    "plt.ylabel(\"FEV\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 5))\n",
    "plt.plot(f_Ntrain_IRNS)\n",
    "plt.plot(f_Ntrain_HEUR)\n",
    "plt.plot(f_Ntrain_FSS)\n",
    "plt.legend([\"IRNS\", \"Heuristics\",\"Full sample size\"], loc =\"upper right\")\n",
    "plt.xlabel(\"Iteration\")\n",
    "plt.ylabel(\"f_Ntrain\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 5))\n",
    "plt.plot(MAX_FEV_IRNS[:-1],f_Ntrain_IRNS)\n",
    "plt.plot(MAX_FEV_HEUR[:-1],f_Ntrain_HEUR)\n",
    "plt.plot(MAX_FEV_FSS[:-1],f_Ntrain_FSS)\n",
    "plt.legend([\"IRNS\", \"Heuristics\",\"Full sample size\"], loc =\"upper right\")\n",
    "plt.xlabel(\"Mushrooms: FEV\")\n",
    "plt.ylabel(\"f_Ntrain\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10, 5))\n",
    "plt.plot(N_IRNS)\n",
    "plt.plot(N_HEUR)\n",
    "plt.plot(N_FSS)\n",
    "plt.legend([\"IRNS\", \"Heuristics\",\"Full sample size\"], loc =\"center left\")\n",
    "plt.xlabel(\"Iteration\")\n",
    "plt.ylabel(\"Sample size\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}