Data Generation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing Classes\n"
     ]
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "from gameClasses import *\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "# from plotConMatrix import plot_confusion_matrix\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.neural_network import MLPRegressor\n",
    "\n",
    "\n",
    "import math as m\n",
    "print(\"Importing Classes\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def movesRandom(state,player):\n",
    "    board = [state,True]\n",
    "    while not(state.isTerminal()) and board[1]:\n",
    "        print(state,board[1])\n",
    "        moves = state.legalMoves(player)\n",
    "        move = random.choice(moves)\n",
    "        board = state.result(move)\n",
    "        \n",
    "        if state.isTerminal() or len(moves) == 1:\n",
    "            break\n",
    "    # print(board[0].board)\n",
    "    return board"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[3 3 4 3 4 4]\n",
      " [5 4 5 4 4 5]]\n",
      "14\n",
      "[[4 3 4 3 4 3]\n",
      " [5 6 4 4 4 4]]\n",
      "32\n",
      "[[4 4 3 3 3 4]\n",
      " [6 4 5 4 4 4]]\n",
      "20\n"
     ]
    }
   ],
   "source": [
    "features = []\n",
    "labels = []\n",
    "\n",
    "\n",
    "\n",
    "for i in range(0,3):\n",
    "    initial = [0,0,np.full((2,6), 4)]\n",
    "    \n",
    "    initial[2][1][random.randint(0,5)] += 1\n",
    "    initial[2][0][random.randint(0,5)] -= 1\n",
    "    initial[2][1][random.randint(0,5)] += 1\n",
    "    initial[2][0][random.randint(0,5)] -= 1\n",
    "    initial[2][1][random.randint(0,5)] += 1\n",
    "    initial[2][0][random.randint(0,5)] -= 1\n",
    "    \n",
    "    print(initial[2])\n",
    "    a = MancalaState(initial)\n",
    "    \n",
    "    aiDepth = random.randint(4,5)\n",
    "    plDepth = random.randint(1,2)\n",
    "    \n",
    "    count = 0\n",
    "    count1 = 0 \n",
    "    while not(a.isTerminal()):\n",
    "        count += 1\n",
    "        a = minimax(a,aiDepth, 0)[0]\n",
    "\n",
    "        if count > 4:\n",
    "    #         print(0,a.board.flatten())\n",
    "            count1 += 1\n",
    "            features += [list(a.board.flatten())]  \n",
    "        if a.isTerminal():\n",
    "            break\n",
    "        #pl1\n",
    "        a = minimax(a,plDepth, 1)[0]\n",
    "        if count > 4:\n",
    "    #         print(1,a.board.flatten())\n",
    "            count1 += 1\n",
    "            features += [list(a.board.flatten())]\n",
    "    labels += [a.aiScore-a.plScore] * count1\n",
    "    print(a.aiScore-a.plScore)\n",
    "    del a\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[4 1 2 ... 3 1 1]\n",
      " [4 1 2 ... 0 0 0]\n",
      " [4 2 3 ... 0 0 0]\n",
      " ...\n",
      " [0 0 1 ... 3 0 0]\n",
      " [0 0 1 ... 4 1 0]\n",
      " [0 0 0 ... 0 0 0]] [-16 -16 -16 ... -28 -28 -28]\n"
     ]
    }
   ],
   "source": [
    "features = np.array(features)\n",
    "labels = np.array(labels)\n",
    "# np.save(\"features\", features)\n",
    "# np.save(\"labels\", labels)\n",
    "print(features,labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = np.load(\"features.npy\")\n",
    "labels = np.load(\"labels.npy\")\n",
    "def learnLogisticalReg(x_train, y_train):\n",
    "    model = LogisticRegression(solver = 'lbfgs',  multi_class ='multinomial')\n",
    "    model.fit(x_train, y_train)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(X_test,y_test,model,title1):\n",
    "    y_pred = model.predict(X_test)\n",
    "    np.set_printoptions(precision=2)\n",
    "    print(np.sum((y_test-y_pred)**2)/len(X_test))\n",
    "    print(title1,\"Accuracy\", accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/mazeyarmoeini/opt/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:947: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
      "  \"of iterations.\", ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "922"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "size = len(features)\n",
    "T_size = m.floor(size*.70)\n",
    "model3 = learnLogisticalReg(features[0:T_size],labels[0:T_size])\n",
    "T_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "115.82212581344902\n",
      "Logistical Regression Accuracy 0.38611713665943603\n"
     ]
    }
   ],
   "source": [
    "evaluate(features[size-T_size:],labels[size-T_size:],model3,\"Logistical Regression\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "def MLP_Regressor(x_train, y_train):\n",
    "    model = MLPRegressor(random_state=0, activation='relu', solver='lbfgs', alpha=1e-5, hidden_layer_sizes=(16, 12, 8, 4))\n",
    "    model.fit(x_train, y_train)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_MLP(X_test,y_test,model,title1):\n",
    "    y_pred = model.predict(X_test)\n",
    "    print(list(zip(y_test,y_pred)))\n",
    "    print(title1,\"Accuracy\", np.sum((y_test-y_pred)**2)/len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "922"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "size = len(features)\n",
    "T_size = m.floor(size*.70)\n",
    "model4 = MLP_Regressor(features[0:T_size],labels[0:T_size])\n",
    "T_size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(-20, -21.2495161516767), (-20, -23.026247984055434), (-20, -22.5489109323518), (-20, -22.44043194513998), (-20, -20.46579079214717), (-20, -25.829855448435353), (-20, -9.767995743758881), (-18, -23.94213938617589), (-18, -18.626137242934014), (-18, -18.62240766709716), (-18, -18.296793951614813), (-18, -17.502153740990064), (-18, -19.014821573899482), (-18, -19.83064724100847), (-18, -18.346785729496464), (-18, -20.832181100518476), (-18, -16.99317375983938), (-18, -14.187068424444448), (-18, -19.886369653641793), (-18, -23.630728690092063), (-18, -24.08459471768205), (-18, -25.246778536332975), (-18, -25.148730044935967), (-18, -25.013586119343394), (-18, -24.409326394683873), (-18, -9.767995743758881), (-16, -15.329772032843163), (-16, -17.013937975465613), (-16, -19.0784884268012), (-16, -18.31001377244507), (-16, -18.38991638699119), (-16, -18.297127644984766), (-16, -9.767995743758881), (-34, -29.560090644374412), (-34, -33.762022225944335), (-34, -34.73054020785156), (-34, -27.419129438719075), (-34, -27.95886739089724), (-34, -32.26665044812435), (-34, -33.540922098949025), (-34, -32.60282361243658), (-34, -33.94028365226226), (-34, -30.092320154732192), (-34, -9.767995743758881), (28, 8.566528769485963), (28, -9.767995743758881), (-16, -18.260776768883602), (-16, -15.501683975444806), (-16, -14.09815963757126), (-16, -16.660131993359034), (-16, -19.161319153893363), (-16, -17.99738328507134), (-16, -19.8586266536908), (-16, -23.382706553934096), (-16, -21.448171068186838), (-16, -12.933002199117041), (-16, -9.767995743758881), (-28, -21.122290410464842), (-28, -26.92733189818243), (-28, -26.134322223666715), (-28, -22.62872506767981), (-28, -23.570847215102646), (-28, -23.529599358595362), (-28, -21.0452816692991), (-28, -23.361998877654557), (-28, -9.767995743758881), (-20, -22.076490915976255), (-20, -18.344972257716698), (-20, -19.441921911758726), (-20, -9.767995743758881), (-26, -20.667997964511798), (-26, -26.03971664293639), (-26, -22.519256665893234), (-26, -20.987707328112844), (-26, -22.511304294813367), (-26, -24.41365414503297), (-26, -25.111086907432032), (-26, -23.388781744182143), (-26, -27.57834775331701), (-26, -25.64461326742576), (-26, -23.993615830231633), (-26, -9.767995743758881), (-34, -30.456297751282925), (-34, -34.65822933285286), (-34, -35.62674731476001), (-34, -29.091041201645865), (-34, -27.95886739089724), (-34, -32.26665044812435), (-34, -33.540922098949025), (-34, -32.60282361243658), (-34, -33.94028365226226), (-34, -30.092320154732192), (-34, -9.767995743758881), (-24, -23.32275363414926), (-24, -21.710411076427032), (-24, -23.97412780237346), (-24, -23.62917026752099), (-24, -24.152102555830112), (-24, -22.042131012237512), (-24, -21.691525800338468), (-24, -20.807738191305294), (-24, -22.917768996843627), (-24, -22.44043194513998), (-24, -23.497584162146005), (-24, -25.829855448435353), (-24, -26.990761159043892), (-24, -9.767995743758881), (-18, -23.94213938617589), (-18, -18.626137242934014), (-18, -18.62240766709716), (-18, -18.296793951614813), (-18, -17.502153740990064), (-18, -19.014821573899482), (-18, -19.83064724100847), (-18, -18.346785729496464), (-18, -20.832181100518476), (-18, -16.99317375983938), (-18, -14.187068424444448), (-18, -19.886369653641793), (-18, -23.630728690092063), (-18, -24.08459471768205), (-18, -25.246778536332975), (-18, -25.148730044935967), (-18, -25.013586119343394), (-18, -24.409326394683873), (-18, -9.767995743758881), (-24, -23.813625452297387), (-24, -22.911150251861475), (-24, -20.44746490291133), (-24, -25.384282398018627), (-24, -20.866563388854487), (-24, -17.985533868556466), (-24, -23.717472711581607), (-24, -22.96689255558611), (-24, -24.727327158323376), (-24, -26.973348131147567), (-24, -9.767995743758881), (14, 8.566528769485963), (14, 8.566528769485963), (14, 8.566528769485963), (14, -9.767995743758881), (-30, -27.746365500525755), (-30, -31.2626544416841), (-30, -30.523019627246406), (-30, -26.777424520763823), (-30, -26.72714689156441), (-30, -25.95754761975455), (-30, -23.44633178229956), (-30, -21.388417116257333), (-30, -25.902821740678455), (-30, -27.323425091234043), (-30, -9.767995743758881), (-28, -22.814475222952733), (-28, -21.450521434871774), (-28, -22.90541440705783), (-28, -21.28762968573115), (-28, -21.382742155616942), (-28, -19.306114167706014), (-28, -31.617351400763837), (-28, -29.994010628185897), (-28, -24.176534152884827), (-28, -25.18838380054484), (-28, -27.728390513618052), (-28, -25.949407897241315), (-28, -9.767995743758881), (-16, -18.260776768883602), (-16, -15.501683975444806), (-16, -14.09815963757126), (-16, -16.660131993359034), (-16, -19.161319153893363), (-16, -17.99738328507134), (-16, -19.8586266536908), (-16, -23.382706553934096), (-16, -21.448171068186838), (-16, -11.144470498419121), (-16, -9.767995743758881), (38, 8.566528769485963), (38, -9.767995743758881), (14, 8.566528769485963), (14, 8.566528769485963), (14, 8.566528769485963), (14, -9.767995743758881), (-18, -23.790425264686135), (-18, -19.614469843208777), (-18, -20.69514969785482), (-18, -9.767995743758881), (2, 8.566528769485963), (2, 8.566528769485963), (2, 8.566528769485963), (2, 8.566528769485963), (2, 8.566528769485963), (2, -9.767995743758881), (18, 8.566528769485963), (18, 8.566528769485963), (18, 8.566528769485963), (18, -9.767995743758881), (-20, -22.076490915976255), (-20, -18.344972257716698), (-20, -19.441921911758726), (-20, -9.767995743758881), (-24, -23.32275363414926), (-24, -21.710411076427032), (-24, -23.97412780237346), (-24, -23.62917026752099), (-24, -24.152102555830112), (-24, -22.042131012237512), (-24, -21.691525800338468), (-24, -20.807738191305294), (-24, -22.917768996843627), (-24, -22.44043194513998), (-24, -23.497584162146005), (-24, -25.829855448435353), (-24, -26.990761159043892), (-24, -9.767995743758881), (28, 8.566528769485963), (28, -9.767995743758881), (-26, -30.477394881994684), (-26, -25.623544996809642), (-26, -25.563836341157113), (-26, -25.330922212879415), (-26, -25.75500922651446), (-26, -28.414307871556325), (-26, -28.561735188633556), (-26, -24.927983407430673), (-26, -24.262094564899613), (-26, -21.436332372406966), (-26, -22.10023567712682), (-26, -23.17103679229748), (-26, -25.552129988670558), (-26, -25.539878683323174), (-26, -19.604998791775948), (-26, -19.262264087383368), (-26, -25.18838380054484), (-26, -22.908047945761794), (-26, -25.949407897241315), (-26, -22.348358800675914), (-26, -9.767995743758881), (-24, -21.122290410464842), (-24, -26.92733189818243), (-24, -26.134322223666715), (-24, -22.62872506767981), (-24, -23.570847215102646), (-24, -21.142752108496598), (-24, -25.064233920848807), (-24, -24.458330420627874), (-24, -9.767995743758881), (12, 8.566528769485963), (12, 8.566528769485963), (12, 8.566528769485963), (12, -9.767995743758881), (-6, -6.764539047244821), (-6, -7.05670012563648), (-6, -6.664539009163947), (-6, -9.767995743758881), (-16, -15.329772032843163), (-16, -17.013937975465613), (-16, -19.0784884268012), (-16, -18.31001377244507), (-16, -18.38991638699119), (-16, -18.297127644984766), (-16, -9.767995743758881), (-20, -20.90041880338502), (-20, -20.175056904099737), (-20, -20.006781465693322), (-20, -20.834514881444573), (-20, -19.2646596376312), (-20, -26.13100883653307), (-20, -24.307532543312128), (-20, -23.240510711867433), (-20, -23.004277881194103), (-20, -21.2495161516767), (-20, -23.026247984055434), (-20, -22.5489109323518), (-20, -22.44043194513998), (-20, -20.46579079214717), (-20, -25.829855448435353), (-20, -9.767995743758881), (-20, -20.90041880338502), (-20, -20.175056904099737), (-20, -20.006781465693322), (-20, -20.834514881444573), (-20, -19.2646596376312), (-20, -26.13100883653307), (-20, -24.307532543312128), (-20, -23.240510711867433), (-20, -23.004277881194103), (-20, -21.2495161516767), (-20, -23.026247984055434), (-20, -22.5489109323518), (-20, -22.44043194513998), (-20, -20.46579079214717), (-20, -25.829855448435353), (-20, -9.767995743758881), (22, 8.566528769485963), (22, -9.767995743758881), (8, 8.566528769485963), (8, 8.566528769485963), (8, 8.566528769485963), (8, 8.566528769485963), (8, 8.566528769485963), (8, 8.566528769485963), (8, 8.566528769485963), (8, -9.767995743758881), (30, 8.566528769485963), (30, 8.566528769485963), (30, 8.566528769485963), (30, -9.767995743758881), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, -9.767995743758881), (-24, -19.303728913884953), (-24, -22.54866204027645), (-24, -26.52058105436859), (-24, -20.226038639410095), (-24, -21.182327112288185), (-24, -20.930623945149975), (-24, -23.058892385134566), (-24, -22.44018670706989), (-24, -22.176454928624864), (-24, -23.304800481135704), (-24, -25.05054605699503), (-24, -23.856265670100008), (-24, -9.767995743758881), (-30, -27.746365500525755), (-30, -31.2626544416841), (-30, -30.523019627246406), (-30, -26.777424520763823), (-30, -26.72714689156441), (-30, -25.95754761975455), (-30, -23.44633178229956), (-30, -21.388417116257333), (-30, -25.902821740678455), (-30, -27.323425091234043), (-30, -9.767995743758881), (-26, -26.978802061548883), (-26, -26.588298384256177), (-26, -28.156740166557263), (-26, -19.077980609179697), (-26, -23.4753130983366), (-26, -24.8514395836783), (-26, -21.60531155427489), (-26, -21.673964909573993), (-26, -24.158057808936228), (-26, -28.278486093118836), (-26, -29.678432288124114), (-26, -26.715523820521035), (-26, -9.767995743758881), (12, 8.566528769485963), (12, 8.566528769485963), (12, 8.566528769485963), (12, -9.767995743758881), (-16, -15.329772032843163), (-16, -17.013937975465613), (-16, -19.0784884268012), (-16, -18.31001377244507), (-16, -18.38991638699119), (-16, -18.297127644984766), (-16, -9.767995743758881), (18, 8.566528769485963), (18, 8.566528769485963), (18, 8.566528769485963), (18, -9.767995743758881), (-18, -21.741964381681218), (-18, -15.97370568438226), (-18, -19.076861281318806), (-18, -16.38059022655829), (-18, -9.767995743758881), (-18, -23.790425264686135), (-18, -19.614469843208777), (-18, -20.69514969785482), (-18, -9.767995743758881), (-30, -27.746365500525755), (-30, -31.2626544416841), (-30, -30.523019627246406), (-30, -26.777424520763823), (-30, -26.72714689156441), (-30, -25.95754761975455), (-30, -23.44633178229956), (-30, -21.388417116257333), (-30, -25.902821740678455), (-30, -27.323425091234043), (-30, -9.767995743758881), (10, 8.566528769485963), (10, -9.767995743758881), (-26, -30.477394881994684), (-26, -25.623544996809642), (-26, -25.563836341157113), (-26, -25.330922212879415), (-26, -25.75500922651446), (-26, -28.414307871556325), (-26, -28.561735188633556), (-26, -24.927983407430673), (-26, -24.262094564899613), (-26, -21.436332372406966), (-26, -22.10023567712682), (-26, -23.17103679229748), (-26, -25.552129988670558), (-26, -25.539878683323174), (-26, -19.604998791775948), (-26, -19.262264087383368), (-26, -25.18838380054484), (-26, -22.908047945761794), (-26, -25.949407897241315), (-26, -22.348358800675914), (-26, -9.767995743758881), (-30, -27.746365500525755), (-30, -31.2626544416841), (-30, -30.523019627246406), (-30, -26.777424520763823), (-30, -26.72714689156441), (-30, -25.95754761975455), (-30, -23.44633178229956), (-30, -21.388417116257333), (-30, -25.902821740678455), (-30, -27.323425091234043), (-30, -9.767995743758881), (2, 4.823644421593881), (2, 8.566528769485963), (2, 8.566528769485963), (2, 8.566528769485963), (2, 8.566528769485963), (2, -9.767995743758881), (-30, -31.161156548681817), (-30, -29.79841142127281), (-30, -32.47654644987596), (-30, -26.62951489542518), (-30, -29.308663547300313), (-30, -31.09888259553494), (-30, -22.83522481968936), (-30, -26.961126872920346), (-30, -9.767995743758881), (-34, -29.560090644374412), (-34, -33.762022225944335), (-34, -34.73054020785156), (-34, -27.419129438719075), (-34, -27.95886739089724), (-34, -32.26665044812435), (-34, -33.540922098949025), (-34, -32.60282361243658), (-34, -33.94028365226226), (-34, -30.092320154732192), (-34, -9.767995743758881), (-20, -17.955971769757003), (-20, -21.8295121956121), (-20, -19.721631152083663), (-20, -21.5751920319178), (-20, -21.526546724385554), (-20, -20.163613992649466), (-20, -18.031293215594673), (-20, -18.743534880718396), (-20, -19.3277225114102), (-20, -14.254729106427757), (-20, -22.53304324553662), (-20, -21.48668698788827), (-20, -24.6243482053509), (-20, -27.0457445586283), (-20, -9.767995743758881), (-34, -29.560090644374412), (-34, -33.762022225944335), (-34, -34.73054020785156), (-34, -27.419129438719075), (-34, -27.95886739089724), (-34, -32.26665044812435), (-34, -33.540922098949025), (-34, -32.60282361243658), (-34, -33.94028365226226), (-34, -30.092320154732192), (-34, -9.767995743758881), (-18, -20.90104736496978), (-18, -19.23611321758011), (-18, -18.45750284978211), (-18, 8.566528769485963), (-18, -17.852296184906045), (-18, -16.766116521261768), (-18, -24.60770231948333), (-18, -19.415474451573008), (-18, -20.10009103692351), (-18, -19.76036400132729), (-18, -20.217141832768238), (-18, -20.10272635387701), (-18, -19.82964253720762), (-18, -20.818694987401006), (-18, -20.373436599598367), (-18, -23.727594770947434), (-18, -25.265046841393513), (-18, -23.38500492299757), (-18, -22.96439677772529), (-18, -14.97176638604563), (-18, -20.179101406008968), (-18, -21.301528049265723), (-18, -22.44096537205685), (-18, -23.28724343930946), (-18, -9.767995743758881), (-30, -15.016400570472712), (-30, -17.42318992514291), (-30, -29.936815308235573), (-30, -23.98794398680878), (-30, -23.838020902788763), (-30, -18.422494547625117), (-30, -22.090239860750337), (-30, -23.452229281639568), (-30, -21.81210565719411), (-30, -19.11441345544774), (-30, -9.767995743758881), (38, 8.566528769485963), (38, -9.767995743758881), (-6, -6.764539047244821), (-6, -7.05670012563648), (-6, -6.664539009163947), (-6, -9.767995743758881), (-26, -26.069569263830452), (-26, -18.054696326858313), (-26, -23.51362072777215), (-26, -24.553692418384465), (-26, -25.382741375519785), (-26, -22.952637520216786), (-26, -25.04219658887792), (-26, -24.108989622824666), (-26, -9.767995743758881), (-14, -13.339372851618045), (-14, -14.120769138279307), (-14, -16.48680197088703), (-14, -20.318264094128544), (-14, -13.542836422536212), (-14, -18.725738366519934), (-14, -22.03823497037804), (-14, -26.028634775330538), (-14, -23.748032887357887), (-14, -22.883833872623427), (-14, -19.196640005036453), (-14, -17.41696402871097), (-14, -20.27109236597584), (-14, -25.002583280265668), (-14, -25.53596484567681), (-14, -22.900231403939813), (-14, -24.409004818398593), (-14, -21.160303755752576), (-14, -23.749387086147117), (-14, -9.767995743758881), (-28, -21.122290410464842), (-28, -26.92733189818243), (-28, -26.134322223666715), (-28, -22.62872506767981), (-28, -23.570847215102646), (-28, -23.529599358595362), (-28, -21.0452816692991), (-28, -23.361998877654557), (-28, -9.767995743758881), (2, 8.566528769485963), (2, 8.566528769485963), (2, 8.566528769485963), (2, 8.566528769485963), (2, 8.566528769485963), (2, -9.767995743758881), (38, 8.566528769485963), (38, -9.767995743758881), (-20, -21.894890910937104), (-20, -27.203219254131547), (-20, -25.05497113583531), (-20, -19.93816618662206), (-20, -22.4938877121088), (-20, -20.731790535885917), (-20, -19.285179069635454), (-20, -19.014429145489583), (-20, -17.758325023402804), (-20, -14.903768597022523), (-20, -15.862954377121792), (-20, -14.513979481597381), (-20, -13.864421297221813), (-20, -10.925704610276153), (-20, -22.709448333492514), (-20, -15.849227782648626), (-20, -18.187321576872247), (-20, -18.950268339950135), (-20, -19.411201413507406), (-20, -9.767995743758881), (-16, -18.260776768883602), (-16, -15.501683975444806), (-16, -14.09815963757126), (-16, -16.660131993359034), (-16, -19.161319153893363), (-16, -17.99738328507134), (-16, -19.8586266536908), (-16, -23.382706553934096), (-16, -21.448171068186838), (-16, -12.933002199117041), (-16, -9.767995743758881), (-18, -20.098560193737107), (-18, -14.07494826067131), (-18, -8.24315705034532), (-18, -8.614987508043157), (-18, -10.035005924520657), (-18, -16.474993565544338), (-18, -15.53760758158841), (-18, -16.536871644036285), (-18, -17.81723740486154), (-18, -16.76648852291362), (-18, -16.661649150355412), (-18, -20.50131469137754), (-18, -19.03536908012774), (-18, -18.92669870486713), (-18, -18.1266882307324), (-18, -15.489138467222936), (-18, -18.799825293597582), (-18, -16.55689955672446), (-18, -12.483813574895244), (-18, -30.410047489258744), (-18, -9.767995743758881), (-6, -6.764539047244821), (-6, -7.05670012563648), (-6, -6.664539009163947), (-6, -9.767995743758881), (-22, -23.635436533475357), (-22, -20.073784149929516), (-22, -21.159847679490216), (-22, -23.50477213024633), (-22, -23.338847009971598), (-22, -21.102168577427026), (-22, -20.026203794714473), (-22, -18.812444203560847), (-22, -20.4034684456388), (-22, -9.767995743758881), (-14, -13.339372851618045), (-14, -14.120769138279307), (-14, -16.48680197088703), (-14, -20.318264094128544), (-14, -13.542836422536212), (-14, -18.725738366519934), (-14, -22.03823497037804), (-14, -26.028634775330538), (-14, -23.748032887357887), (-14, -22.883833872623427), (-14, -19.196640005036453), (-14, -17.41696402871097), (-14, -20.27109236597584), (-14, -25.002583280265668), (-14, -25.53596484567681), (-14, -22.900231403939813), (-14, -24.409004818398593), (-14, -21.160303755752576), (-14, -23.749387086147117), (-14, -9.767995743758881), (-6, -6.764539047244821), (-6, -7.05670012563648), (-6, -6.664539009163947), (-6, -9.767995743758881), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, -9.767995743758881), (-22, -23.635436533475357), (-22, -20.073784149929516), (-22, -21.159847679490216), (-22, -23.50477213024633), (-22, -23.338847009971598), (-22, -21.102168577427026), (-22, -20.026203794714473), (-22, -18.812444203560847), (-22, -20.4034684456388), (-22, -9.767995743758881), (-26, -22.790629381698963), (-26, -16.054309551324266), (-26, -20.23434733727917), (-26, -17.378786697913466), (-26, -11.964542024100723), (-26, -10.491263684532427), (-26, -13.136301689959211), (-26, -21.39008000100853), (-26, -22.59608009586834), (-26, -22.064554694926173), (-26, -9.767995743758881), (14, 8.566528769485963), (14, 8.566528769485963), (14, 8.566528769485963), (14, -9.767995743758881), (-34, -30.456297751282925), (-34, -34.65822933285286), (-34, -35.62674731476001), (-34, -29.091041201645865), (-34, -27.95886739089724), (-34, -32.26665044812435), (-34, -33.540922098949025), (-34, -32.60282361243658), (-34, -33.94028365226226), (-34, -30.092320154732192), (-34, -9.767995743758881), (-20, -21.894890910937104), (-20, -27.203219254131547), (-20, -25.05497113583531), (-20, -19.93816618662206), (-20, -22.4938877121088), (-20, -20.731790535885917), (-20, -19.285179069635454), (-20, -19.014429145489583), (-20, -17.758325023402804), (-20, -14.903768597022523), (-20, -15.862954377121792), (-20, -14.513979481597381), (-20, -13.864421297221813), (-20, -10.925704610276153), (-20, -22.709448333492514), (-20, -15.849227782648626), (-20, -18.187321576872247), (-20, -18.950268339950135), (-20, -19.411201413507406), (-20, -9.767995743758881), (30, 8.566528769485963), (30, 8.566528769485963), (30, 8.566528769485963), (30, -9.767995743758881), (14, 8.566528769485963), (14, 8.566528769485963), (14, 8.566528769485963), (14, -9.767995743758881), (-10, -19.422358913341903), (-10, -15.877722451930001), (-10, -16.128955718270785), (-10, -16.915398687120927), (-10, -15.7031064668064), (-10, -10.807325548738351), (-10, -24.255774437227515), (-10, -21.216061171384307), (-10, -19.72806120949237), (-10, -21.074474396663437), (-10, -20.60735682111856), (-10, -15.083913367557692), (-10, -16.377911812265143), (-10, -16.31346426151316), (-10, -14.358121495295524), (-10, -9.767995743758881), (-28, -21.122290410464842), (-28, -26.92733189818243), (-28, -26.134322223666715), (-28, -22.62872506767981), (-28, -23.570847215102646), (-28, -23.529599358595362), (-28, -21.0452816692991), (-28, -23.361998877654557), (-28, -9.767995743758881), (22, 8.566528769485963), (22, -9.767995743758881), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, 8.566528769485963), (20, -9.767995743758881), (-20, -17.955971769757003), (-20, -21.8295121956121), (-20, -19.721631152083663), (-20, -21.5751920319178), (-20, -21.526546724385554), (-20, -20.163613992649466), (-20, -18.031293215594673), (-20, -18.743534880718396), (-20, -19.3277225114102), (-20, -14.254729106427757), (-20, -22.53304324553662), (-20, -21.48668698788827), (-20, -24.6243482053509), (-20, -27.045744558628314), (-20, -9.767995743758881), (-6, -6.764539047244821), (-6, -7.05670012563648), (-6, -6.664539009163947), (-6, -9.767995743758881), (4, -22.979311117320044), (4, 8.566528769485963), (4, 8.566528769485963), (4, 8.566528769485963), (4, 8.566528769485963), (4, 8.566528769485963), (4, 8.566528769485963), (4, 8.566528769485963), (4, 8.566528769485963), (4, -9.767995743758881), (-34, -30.456297751282925), (-34, -34.65822933285286), (-34, -35.62674731476001), (-34, -29.091041201645865), (-34, -27.95886739089724), (-34, -32.26665044812435), (-34, -33.540922098949025), (-34, -32.60282361243658), (-34, -33.94028365226226), (-34, -30.092320154732192), (-34, -9.767995743758881), (8, -14.945658175911035), (8, -16.245778429402687), (8, -16.900979025135932), (8, -14.209166821229214), (8, -15.042565989508624), (8, -13.157752940180263), (8, 8.566528769485963), (8, -9.767995743758881), (22, 8.566528769485963), (22, -9.767995743758881), (-6, -6.764539047244821), (-6, -7.05670012563648), (-6, -6.664539009163947), (-6, -9.767995743758881), (-24, -23.813625452297387), (-24, -22.911150251861475), (-24, -20.44746490291133), (-24, -25.384282398018627), (-24, -20.866563388854487), (-24, -17.985533868556466), (-24, -23.717472711581607), (-24, -22.96689255558611), (-24, -24.727327158323376), (-24, -26.973348131147567), (-24, -9.767995743758881), (38, 8.566528769485963), (38, -9.767995743758881), (-20, -17.955971769757003), (-20, -21.8295121956121), (-20, -19.721631152083663), (-20, -21.5751920319178), (-20, -21.526546724385554), (-20, -20.163613992649466), (-20, -18.031293215594673), (-20, -18.743534880718396), (-20, -19.3277225114102), (-20, -14.254729106427757), (-20, -22.53304324553662), (-20, -21.48668698788827), (-20, -24.6243482053509), (-20, -27.045744558628314), (-20, -9.767995743758881), (22, 8.566528769485963), (22, 8.566528769485963), (22, 8.566528769485963), (22, -9.767995743758881), (-30, -15.016400570472712), (-30, -17.42318992514291), (-30, -29.936815308235573), (-30, -23.98794398680878), (-30, -23.838020902788763), (-30, -18.422494547625117), (-30, -22.090239860750337), (-30, -23.452229281639568), (-30, -21.81210565719411), (-30, -19.11441345544774), (-30, -9.767995743758881), (6, -29.659910557565574), (6, -25.576366659027805), (6, -23.23940720310428), (6, -23.30467824678561), (6, -22.110360728061455), (6, -14.131419447486827), (6, -14.005861876553002), (6, -9.767995743758881), (-26, -30.477394881994684), (-26, -25.623544996809642), (-26, -25.563836341157113), (-26, -25.330922212879415), (-26, -25.75500922651446), (-26, -28.414307871556325), (-26, -28.561735188633556), (-26, -24.927983407430673), (-26, -24.262094564899613), (-26, -21.436332372406966), (-26, -22.10023567712682), (-26, -23.17103679229748), (-26, -25.552129988670558), (-26, -25.539878683323174), (-26, -19.604998791775948), (-26, -19.262264087383368), (-26, -25.18838380054484), (-26, -22.908047945761794), (-26, -25.949407897241315), (-26, -22.348358800675914), (-26, -9.767995743758881), (-4, -21.10955926617433), (-4, -25.693897127290597), (-4, -21.466688477237213), (-4, -21.39226106243891), (-4, -25.580889442926846), (-4, -19.704153337531018), (-4, -15.8248736091893), (-4, -15.422130602174647), (-4, -15.319885791039358), (-4, -9.831510807013064), (-4, -9.767995743758881), (-26, -26.069569263830452), (-26, -18.054696326858313), (-26, -23.51362072777215), (-26, -24.553692418384465), (-26, -25.382741375519785), (-26, -22.952637520216786), (-26, -25.04219658887792), (-26, -24.108989622824666), (-26, -9.767995743758881), (-30, -31.161156548681817), (-30, -29.79841142127281), (-30, -32.47654644987596), (-30, -26.62951489542518), (-30, -29.308663547300313), (-30, -31.09888259553494), (-30, -22.83522481968936), (-30, -26.961126872920346), (-30, -9.767995743758881), (-28, -21.122290410464842), (-28, -26.92733189818243), (-28, -26.134322223666715), (-28, -22.62872506767981), (-28, -23.570847215102646), (-28, -23.529599358595362), (-28, -21.0452816692991), (-28, -23.361998877654557), (-28, -9.767995743758881)]\n",
      "MLP Regression Accuracy 87.88483203412814\n"
     ]
    }
   ],
   "source": [
    "evaluate_MLP(features[size-T_size:],labels[size-T_size:],model4,\"MLP Regression\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}