FeatureSelection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "faf8332b",
   "metadata": {},
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c3ebd3e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train (191, 9) (191,)\n",
      "Test (95, 9) (95,)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OrdinalEncoder, LabelEncoder\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    "def prepare_inputs(xtrain, xtest):\n",
    "    oe = OrdinalEncoder()\n",
    "    oe.fit(xtrain)\n",
    "    xtrain_enc = oe.transform(xtrain)\n",
    "    xtest_enc = oe.transform(xtest)\n",
    "    \n",
    "    return xtrain_enc, xtest_enc\n",
    "\n",
    "def prepare_targets(ytrain, ytest):\n",
    "    le = LabelEncoder()\n",
    "    le.fit(ytrain)\n",
    "    ytrain_enc = le.transform(ytrain)\n",
    "    ytest_enc = le.transform(ytest)\n",
    "    \n",
    "    return ytrain_enc, ytest_enc\n",
    "\n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\BreastCancer.csv\")\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_enc, xtest_enc = prepare_inputs(xtrain, xtest)\n",
    "ytrain_enc, ytest_enc = prepare_targets(ytrain, ytest)\n",
    "\n",
    "print('Train', xtrain_enc.shape, ytrain_enc.shape)\n",
    "print('Test', xtest_enc.shape, ytest_enc.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5be54a3a",
   "metadata": {},
   "source": [
    "# Categorical input data when the target variable is also categorical (Categorical Feature Selection) :"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ab90920",
   "metadata": {},
   "source": [
    "## Chi-Squared Feature Selection:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2cb99abd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature 0: 0.472553\n",
      "Feature 1: 0.029193\n",
      "Feature 2: 2.137658\n",
      "Feature 3: 29.381059\n",
      "Feature 4: 8.222601\n",
      "Feature 5: 8.100183\n",
      "Feature 6: 1.273822\n",
      "Feature 7: 0.950682\n",
      "Feature 8: 3.699989\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA54AAAHSCAYAAACEihTZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAUZ0lEQVR4nO3dUaik91nH8d9jNlJtK6bkNMQkuFVCNQhJZInRQKlNq6mKaS8KCRiCVNaLRltpkNgb9c6LWPVCCmsbGzA2lLbSIEENsSJCiW7apE3clpQa07RrcqpooxfWtI8XZwrb7YZz9px5djKTzwcOM+877+w8Fy/L+e4773+ruwMAAABTvmvVAwAAALDZhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjDp3LD7vwwgv78OHD5/IjAQAAOEcefvjhr3b31un7z2l4Hj58OMePHz+XHwkAAMA5UlX/eqb9vmoLAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAqF3Ds6peVlX/WFWPVtXjVfW7i/2vqqoHquqJxeMF8+MCAACwbvZyxfN/k7yhu69MclWSG6rq2iR3JHmwuy9P8uBiGwAAAL7NruHZO/57sXn+4qeT3Jjk7sX+u5O8ZWJAAAAA1tue7vGsqvOq6pEkzyZ5oLsfSnJRd59MksXjq8emBAAAYG3tKTy7+xvdfVWSS5NcU1U/ttcPqKqjVXW8qo5vb2/vc0wAAADW1Vmtatvd/5nk75LckOSZqro4SRaPz77Ae45195HuPrK1tXWwaQEAAFg7e1nVdquqvn/x/HuSvDHJ55Lcl+TWxWG3Jvn40IwAAACssUN7OObiJHdX1XnZCdUPd/dfVtUnk3y4qt6e5KkkbxucEwAAgDW1a3h292eSXH2G/f+e5PqJoQAAANgcZ3WPJwAAAJwt4QkAAMAo4QkAAMAo4QkAAMCovaxqC8A+XHnnTaseYS08evu9qx4BABjmiicAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACjdg3Pqrqsqj5RVSeq6vGqeudi/+9U1Zer6pHFz8/NjwsAAMC6ObSHY55P8u7u/lRVvTLJw1X1wOK1P+juO+fGAwAAYN3tGp7dfTLJycXz56rqRJJLpgcDAABgM5zVPZ5VdTjJ1UkeWuy6rao+U1V3VdUFL/Ceo1V1vKqOb29vH2xaAAAA1s6ew7OqXpHko0ne1d1fS/K+JD+c5KrsXBH9/TO9r7uPdfeR7j6ytbV18IkBAABYK3sKz6o6PzvReU93fyxJuvuZ7v5Gd38zyZ8kuWZuTAAAANbVXla1rSQfSHKiu997yv6LTznsrUkeW/54AAAArLu9rGp7XZJbkny2qh5Z7HtPkpur6qokneTJJL86MB8AAABrbi+r2v5DkjrDS/cvfxwAAAA2zVmtagsAAABnS3gCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwSngCAAAwatfwrKrLquoTVXWiqh6vqncu9r+qqh6oqicWjxfMjwsAAMC62csVz+eTvLu7fzTJtUneUVVXJLkjyYPdfXmSBxfbAAAA8G12Dc/uPtndn1o8fy7JiSSXJLkxyd2Lw+5O8pahGQEAAFhjZ3WPZ1UdTnJ1koeSXNTdJ5OdOE3y6qVPBwAAwNrbc3hW1SuSfDTJu7r7a2fxvqNVdbyqjm9vb+9nRgAAANbYnsKzqs7PTnTe090fW+x+pqouXrx+cZJnz/Te7j7W3Ue6+8jW1tYyZgYAAGCN7GVV20rygSQnuvu9p7x0X5JbF89vTfLx5Y8HAADAuju0h2OuS3JLks9W1SOLfe9J8ntJPlxVb0/yVJK3jUwIAADAWts1PLv7H5LUC7x8/XLHAQAAYNOc1aq2AAAAcLaEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKOEJwAAAKN2Dc+ququqnq2qx07Z9ztV9eWqemTx83OzYwIAALCu9nLF84NJbjjD/j/o7qsWP/cvdywAAAA2xa7h2d1/n+Q/zsEsAAAAbKCD3ON5W1V9ZvFV3Ate6KCqOlpVx6vq+Pb29gE+DgAAgHW03/B8X5IfTnJVkpNJfv+FDuzuY919pLuPbG1t7fPjAAAAWFf7Cs/ufqa7v9Hd30zyJ0muWe5YAAAAbIp9hWdVXXzK5luTPPZCxwIAAPDSdmi3A6rqQ0len+TCqno6yW8neX1VXZWkkzyZ5FfnRgQAAGCd7Rqe3X3zGXZ/YGAWAAAANtBBVrUFAACAXQlPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARglPAAAARu0anlV1V1U9W1WPnbLvVVX1QFU9sXi8YHZMAAAA1tVernh+MMkNp+27I8mD3X15kgcX2wAAAPAddg3P7v77JP9x2u4bk9y9eH53krcsdywAAAA2xX7v8byou08myeLx1csbCQAAgE0yvrhQVR2tquNVdXx7e3v64wAAAHiR2W94PlNVFyfJ4vHZFzqwu49195HuPrK1tbXPjwMAAGBd7Tc870ty6+L5rUk+vpxxAAAA2DR7+e9UPpTkk0leW1VPV9Xbk/xekjdV1RNJ3rTYBgAAgO9waLcDuvvmF3jp+iXPAgAAwAYaX1wIAACAlzbhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwCjhCQAAwKhDqx4AAJbhyjtvWvUIa+HR2+9d9QgAvAS54gkAAMAoVzwBgH1xlXl3rjAD7HDFEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFHCEwAAgFGHDvLmqnoyyXNJvpHk+e4+soyhAAAA2BwHCs+Fn+7ury7hzwEAAGAD+aotAAAAow4anp3kb6rq4ao6eqYDqupoVR2vquPb29sH/DgAAADWzUHD87ru/vEkb07yjqp63ekHdPex7j7S3Ue2trYO+HEAAACsmwOFZ3d/ZfH4bJK/SHLNMoYCAABgc+w7PKvq5VX1ym89T/IzSR5b1mAAAABshoOsantRkr+oqm/9OX/e3X+1lKkAAADYGPsOz+7+YpIrlzgLAAAAG8h/pwIAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMCoQ6seAAAAeGm78s6bVj3CWnj09ntXPcK+ueIJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAqEOrHgA4d66886ZVj7AWHr393lWPAACwUVzxBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYNShVQ8AAMDurrzzplWPsBYevf3eVY8AnIErngAAAIwSngAAAIwSngAAAIxyjycAAJzGPbV7455a9soVTwAAAEYJTwAAAEYJTwAAAEYJTwAAAEYJTwAAAEYJTwAAAEb571ROY+ns3Vk2GwAAOBuueAIAADBKeAIAADDqQOFZVTdU1eer6gtVdceyhgIAAGBz7Ds8q+q8JH+c5M1Jrkhyc1VdsazBAAAA2AwHueJ5TZIvdPcXu/vrSe5NcuNyxgIAAGBTHGRV20uSfOmU7aeT/MTBxuGlxirCe2MlYQAA1ll19/7eWPW2JD/b3b+y2L4lyTXd/WunHXc0ydHF5muTfH7/474kXZjkq6seAgY5x9lkzm82nXOcTeb83p8f7O6t03ce5Irn00kuO2X70iRfOf2g7j6W5NgBPuclraqOd/eRVc8BU5zjbDLnN5vOOc4mc34v10Hu8fynJJdX1Wuq6ruT3JTkvuWMBQAAwKbY9xXP7n6+qm5L8tdJzktyV3c/vrTJAAAA2AgH+aptuvv+JPcvaRbOzNeU2XTOcTaZ85tN5xxnkzm/l2jfiwsBAADAXhzkHk8AAADYlfB8EauqG6rq81X1haq6Y9XzwLJU1WVV9YmqOlFVj1fVO1c9E0yoqvOq6tNV9ZerngWWqaq+v6o+UlWfW/xd/pOrngmWqap+Y/E7ymNV9aGqetmqZ1p3wvNFqqrOS/LHSd6c5IokN1fVFaudCpbm+STv7u4fTXJtknc4v9lQ70xyYtVDwIA/SvJX3f0jSa6M85wNUlWXJPn1JEe6+8eys5DqTaudav0Jzxeva5J8obu/2N1fT3JvkhtXPBMsRXef7O5PLZ4/l51fWC5Z7VSwXFV1aZKfT/L+Vc8Cy1RV35fkdUk+kCTd/fXu/s+VDgXLdyjJ91TVoSTfm+QrK55n7QnPF69LknzplO2n4xdzNlBVHU5ydZKHVjwKLNsfJvnNJN9c8RywbD+UZDvJny6+Sv7+qnr5qoeCZenuLye5M8lTSU4m+a/u/pvVTrX+hOeLV51hnyWI2ShV9YokH03yru7+2qrngWWpql9I8mx3P7zqWWDAoSQ/nuR93X11kv9JYi0KNkZVXZCdbxq+JskPJHl5Vf3Saqdaf8LzxevpJJedsn1pXOJng1TV+dmJznu6+2OrngeW7Lokv1hVT2bnVok3VNWfrXYkWJqnkzzd3d/6pspHshOisCnemORfunu7u/8vyceS/NSKZ1p7wvPF65+SXF5Vr6mq787ODc33rXgmWIqqquzcG3Siu9+76nlg2br7t7r70u4+nJ2/v/+2u/1rORuhu/8tyZeq6rWLXdcn+ecVjgTL9lSSa6vqexe/s1wfC2gd2KFVD8CZdffzVXVbkr/Ozkpad3X34yseC5bluiS3JPlsVT2y2Pee7r5/dSMBcBZ+Lck9i38c/2KSX17xPLA03f1QVX0kyaeysxL/p5McW+1U66+63TYIAADAHF+1BQAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYNT/AyJm7H0yaUXxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1152x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OrdinalEncoder, LabelEncoder\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    "def prepare_inputs(xtrain, xtest):\n",
    "    oe = OrdinalEncoder()\n",
    "    oe.fit(xtrain)\n",
    "    xtrain_enc = oe.transform(xtrain)\n",
    "    xtest_enc = oe.transform(xtest)\n",
    "    \n",
    "    return xtrain_enc, xtest_enc\n",
    "\n",
    "def prepare_targets(ytrain, ytest):\n",
    "    le = LabelEncoder()\n",
    "    le.fit(ytrain)\n",
    "    ytrain_enc = le.transform(ytrain)\n",
    "    ytest_enc = le.transform(ytest)\n",
    "    \n",
    "    return ytrain_enc, ytest_enc\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=chi2, k='all')\n",
    "    fs.fit(xtrain, ytrain)\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    \n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\BreastCancer.csv\")\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_enc, xtest_enc = prepare_inputs(xtrain, xtest)\n",
    "ytrain_enc, ytest_enc = prepare_targets(ytrain, ytest)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain_enc, ytrain_enc, xtest_enc)\n",
    "\n",
    "for i in range(len(fs.scores_)):\n",
    "    print('Feature %d: %f' % (i, fs.scores_[i]))\n",
    "\n",
    "plt.figure(figsize=(16,8))    \n",
    "plt.bar([i for i in range(len(fs.scores_))], fs.scores_, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b2f6976",
   "metadata": {},
   "source": [
    "## Mutual Information Feature Selection:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "07b7c702",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature 0: 0.019654\n",
      "Feature 1: 0.000116\n",
      "Feature 2: 0.085856\n",
      "Feature 3: 0.000000\n",
      "Feature 4: 0.051495\n",
      "Feature 5: 0.070729\n",
      "Feature 6: 0.025941\n",
      "Feature 7: 0.004189\n",
      "Feature 8: 0.008768\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6gAAAHSCAYAAADhZ+amAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVvklEQVR4nO3dUahdd1r38d9jjkVnRCpMLmpSTAfCaBDClFCjA144wttUMVcvpDBTKEgotOMolqF64+V7E0QLpaHMVCgO5qLORdBghVe98KKl6XRarbEQqk5jI8YLq9iLGnzei7NfOJ45nex2dtzP2f184MBZa/1XzrNhEc43a+2d6u4AAADAun3fugcAAACARKACAAAwhEAFAABgBIEKAADACAIVAACAEQQqAAAAI2yte4C9fOpTn+ojR46sewwAAABW7JVXXvmX7j6417GRgXrkyJFcvnx53WMAAACwYlX1Dx90zCO+AAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwwta6BwDmOX7uzLpHGO+1xy+sewQAgI3jDioAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwgkAFAABgBIEKAADACAIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAISwVqVd1fVW9W1dWqemKP41VVTy6Ov15V9+449mtV9UZV/XVV/UFV/cAqXwAAAACb4ZaBWlUHkjyV5FSSY0kerKpju5adSnJ08XU2ydOLcw8l+ZUkJ7r7J5McSHJmZdMDAACwMZa5g3pfkqvd/VZ3v5/kQpLTu9acTvJcb3sxyZ1Vddfi2FaSH6yqrSSfSPLOimYHAABggywTqIeSvL1j+9pi3y3XdPc/JjmX5NtJrid5t7v/9KOPCwAAwKZaJlBrj329zJqq+pFs3129J8mPJvlkVX1hzx9SdbaqLlfV5Rs3biwxFgAAAJtkmUC9luTuHduH852P6X7Qmp9P8nfdfaO7/zPJN5L8zF4/pLuf6e4T3X3i4MGDy84PAADAhlgmUF9OcrSq7qmqO7L9IUcXd625mOShxaf5nsz2o7zXs/1o78mq+kRVVZLPJ7mywvkBAADYEFu3WtDdN6vqsSQvZPtTeJ/t7jeq6pHF8fNJLiV5IMnVJO8leXhx7KWqej7JN5PcTPJqkmduxwsBAABgf7tloCZJd1/KdoTu3Hd+x/ed5NEPOPe3kvzW9zAjAAAAHwPLPOILAAAAt51ABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwgkAFAABgBIEKAADACAIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYYWvdAwAAm+v4uTPrHmFfeO3xC+seAWAEd1ABAAAYQaACAAAwgkAFAABgBIEKAADACAIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwgkAFAABgBIEKAADACAIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGWCpQq+r+qnqzqq5W1RN7HK+qenJx/PWqunfHsTur6vmq+tuqulJVP73KFwAAAMBmuGWgVtWBJE8lOZXkWJIHq+rYrmWnkhxdfJ1N8vSOY7+b5E+6+8eTHE9yZQVzAwAAsGGWuYN6X5Kr3f1Wd7+f5EKS07vWnE7yXG97McmdVXVXVf1wkp9N8rUk6e73u/tfVzc+AAAAm2KZQD2U5O0d29cW+5ZZ8+kkN5L8XlW9WlVfrapP7vVDqupsVV2uqss3btxY+gUAAACwGZYJ1NpjXy+5ZivJvUme7u7PJvmPJN/xHtYk6e5nuvtEd584ePDgEmMBAACwSZYJ1GtJ7t6xfTjJO0uuuZbkWne/tNj/fLaDFQAAAP6bZQL15SRHq+qeqrojyZkkF3etuZjkocWn+Z5M8m53X+/uf0rydlV9ZrHu80n+ZlXDAwAAsDm2brWgu29W1WNJXkhyIMmz3f1GVT2yOH4+yaUkDyS5muS9JA/v+CO+lOTri7h9a9cxAAAASLJEoCZJd1/KdoTu3Hd+x/ed5NEPOPdbSU589BEBAAD4OFjmEV8AAAC47QQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwgkAFAABgBIEKAADACAIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEbYWvcAAPA/6fi5M+seYV947fEL6x4BgI8hd1ABAAAYQaACAAAwgkAFAABgBIEKAADACAIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwgkAFAABgBIEKAADACAIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwgkAFAABghKUCtarur6o3q+pqVT2xx/GqqicXx1+vqnt3HT9QVa9W1R+tanAAAAA2yy0DtaoOJHkqyakkx5I8WFXHdi07leTo4utskqd3Hf9ykivf87QAAABsrGXuoN6X5Gp3v9Xd7ye5kOT0rjWnkzzX215McmdV3ZUkVXU4yS8k+eoK5wYAAGDDLBOoh5K8vWP72mLfsmt+J8lXkvzXRxsRAACAj4NlArX22NfLrKmqX0zyz939yi1/SNXZqrpcVZdv3LixxFgAAABskmUC9VqSu3dsH07yzpJrPpfkl6rq77P9aPDPVdXv7/VDuvuZ7j7R3ScOHjy45PgAAABsimUC9eUkR6vqnqq6I8mZJBd3rbmY5KHFp/meTPJud1/v7t/o7sPdfWRx3p919xdW+QIAAADYDFu3WtDdN6vqsSQvJDmQ5NnufqOqHlkcP5/kUpIHklxN8l6Sh2/fyAAAAGyiWwZqknT3pWxH6M5953d830kevcWf8RdJ/uJDTwgAAMDHwjKP+AIAAMBtJ1ABAAAYQaACAAAwgkAFAABgBIEKAADACAIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwgkAFAABgBIEKAADACAIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwgkAFAABgBIEKAADACAIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACNsrXsAAABW5/i5M+seYbzXHr+w7hGAD+AOKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwgkAFAABgBIEKAADACAIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBGWCtSqur+q3qyqq1X1xB7Hq6qeXBx/varuXey/u6r+vKquVNUbVfXlVb8AAAAANsMtA7WqDiR5KsmpJMeSPFhVx3YtO5Xk6OLrbJKnF/tvJvn17v6JJCeTPLrHuQAAALDUHdT7klzt7re6+/0kF5Kc3rXmdJLnetuLSe6sqru6+3p3fzNJuvvfk1xJcmiF8wMAALAhlgnUQ0ne3rF9Ld8ZmbdcU1VHknw2yUt7/ZCqOltVl6vq8o0bN5YYCwAAgE2yTKDWHvv6w6ypqh9K8odJfrW7/22vH9Ldz3T3ie4+cfDgwSXGAgAAYJNsLbHmWpK7d2wfTvLOsmuq6vuzHadf7+5vfPRR5zh+7sy6R9gXXnv8wrpHAAAA9pFl7qC+nORoVd1TVXckOZPk4q41F5M8tPg035NJ3u3u61VVSb6W5Ep3//ZKJwcAAGCj3PIOanffrKrHkryQ5ECSZ7v7jap6ZHH8fJJLSR5IcjXJe0keXpz+uSRfTPJXVfWtxb7f7O5LK30VAAAA7HvLPOKbRVBe2rXv/I7vO8mje5z3l9n7/akAAADw3yzziC8AAADcdgIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwgkAFAABgBIEKAADACAIVAACAEQQqAAAAIwhUAAAARhCoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwgkAFAABgBIEKAADACFvrHgAAAGAZx8+dWfcI+8Jrj19Y9wgfmTuoAAAAjCBQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGEKgAAACMIFABAAAYQaACAAAwgkAFAABgBIEKAADACFvrHgAAAPar4+fOrHuEfeG1xy+sewT2CXdQAQAAGEGgAgAAMIJABQAAYASBCgAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGAEgQoAAMAIAhUAAIARBCoAAAAjCFQAAABGWCpQq+r+qnqzqq5W1RN7HK+qenJx/PWqunfZcwEAACBZIlCr6kCSp5KcSnIsyYNVdWzXslNJji6+ziZ5+kOcCwAAAEvdQb0vydXufqu7309yIcnpXWtOJ3mut72Y5M6qumvJcwEAACBbS6w5lOTtHdvXkvzUEmsOLXkufFfHz51Z9wj7wmuPX1j3CAAA8D2p7v7uC6r+d5L/1d2/vNj+YpL7uvtLO9b8cZL/091/udj+v0m+kuTTtzp3x59xNtuPByfJZ5K8+T2+to+bTyX5l3UPAbeRa5xN5vpmk7m+2XSu8Q/vx7r74F4HlrmDei3J3Tu2Dyd5Z8k1dyxxbpKku59J8swS87CHqrrc3SfWPQfcLq5xNpnrm03m+mbTucZXa5n3oL6c5GhV3VNVdyQ5k+TirjUXkzy0+DTfk0ne7e7rS54LAAAAt76D2t03q+qxJC8kOZDk2e5+o6oeWRw/n+RSkgeSXE3yXpKHv9u5t+WVAAAAsK8t84hvuvtStiN0577zO77vJI8uey63hcej2XSucTaZ65tN5vpm07nGV+iWH5IEAAAA/xOWeQ8qAAAA3HYCdZ+rqvur6s2qulpVT6x7Hlilqrq7qv68qq5U1RtV9eV1zwSrVlUHqurVqvqjdc8Cq1ZVd1bV81X1t4u/y3963TPBqlTVry1+P/nrqvqDqvqBdc+0CQTqPlZVB5I8leRUkmNJHqyqY+udClbqZpJf7+6fSHIyyaOucTbQl5NcWfcQcJv8bpI/6e4fT3I8rnU2RFUdSvIrSU50909m+wNhz6x3qs0gUPe3+5Jc7e63uvv9JBeSnF7zTLAy3X29u7+5+P7fs/2LzaH1TgWrU1WHk/xCkq+uexZYtar64SQ/m+RrSdLd73f3v651KFitrSQ/WFVbST6R5J01z7MRBOr+dijJ2zu2r8Uv72yoqjqS5LNJXlrzKLBKv5PkK0n+a81zwO3w6SQ3kvze4jH2r1bVJ9c9FKxCd/9jknNJvp3kepJ3u/tP1zvVZhCo+1vtsc/HMrNxquqHkvxhkl/t7n9b9zywClX1i0n+ubtfWfcscJtsJbk3ydPd/dkk/5HE52WwEarqR7L95OI9SX40ySer6gvrnWozCNT97VqSu3dsH45HC9gwVfX92Y7Tr3f3N9Y9D6zQ55L8UlX9fbbfovFzVfX76x0JVupakmvd/f+ffHk+28EKm+Dnk/xdd9/o7v9M8o0kP7PmmTaCQN3fXk5ytKruqao7sv3G7ItrnglWpqoq2+9dutLdv73ueWCVuvs3uvtwdx/J9t/ff9bd/vWdjdHd/5Tk7ar6zGLX55P8zRpHglX6dpKTVfWJxe8rn48PAVuJrXUPwEfX3Ter6rEkL2T7k8Oe7e431jwWrNLnknwxyV9V1bcW+36zuy+tbyQAPoQvJfn64h/S30ry8JrngZXo7peq6vkk38z2/zrwapJn1jvVZqhub1kEAABg/TziCwAAwAgCFQAAgBEEKgAAACMIVAAAAEYQqAAAAIwgUAEAABhBoAIAADCCQAUAAGCE/wfkSpyGmrhtbQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OrdinalEncoder, LabelEncoder\n",
    "from sklearn.feature_selection import SelectKBest,chi2, mutual_info_classif\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    "def prepare_inputs(xtrain, xtest):\n",
    "    oe = OrdinalEncoder()\n",
    "    oe.fit(xtrain)\n",
    "    xtrain_enc = oe.transform(xtrain)\n",
    "    xtest_enc = oe.transform(xtest)\n",
    "    \n",
    "    return xtrain_enc, xtest_enc\n",
    "\n",
    "def prepare_targets(ytrain, ytest):\n",
    "    le = LabelEncoder()\n",
    "    le.fit(ytrain)\n",
    "    ytrain_enc = le.transform(ytrain)\n",
    "    ytest_enc = le.transform(ytest)\n",
    "    \n",
    "    return ytrain_enc, ytest_enc\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=mutual_info_classif, k='all')\n",
    "    fs.fit(xtrain, ytrain)\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    \n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\BreastCancer.csv\")\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_enc, xtest_enc = prepare_inputs(xtrain, xtest)\n",
    "ytrain_enc, ytest_enc = prepare_targets(ytrain, ytest)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain_enc, ytrain_enc, xtest_enc)\n",
    "\n",
    "for i in range(len(fs.scores_)):\n",
    "    print('Feature %d: %f' % (i, fs.scores_[i]))\n",
    "\n",
    "plt.figure(figsize=(16, 8))    \n",
    "plt.bar([i for i in range(len(fs.scores_))], fs.scores_, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "598a32e5",
   "metadata": {},
   "source": [
    "## Model built using all Features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "042a9999",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 75.79\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OrdinalEncoder, LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    "def prepare_inputs(xtrain, xtest):\n",
    "    oe = OrdinalEncoder()\n",
    "    oe.fit(xtrain)\n",
    "    xtrain_enc = oe.transform(xtrain)\n",
    "    xtest_enc = oe.transform(xtest)\n",
    "    \n",
    "    return xtrain_enc, xtest_enc\n",
    "\n",
    "def prepare_targets(ytrain, ytest):\n",
    "    le = LabelEncoder()\n",
    "    le.fit(ytrain)\n",
    "    ytrain_enc = le.transform(ytrain)\n",
    "    ytest_enc = le.transform(ytest)\n",
    "    \n",
    "    return ytrain_enc, ytest_enc\n",
    "\n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\BreastCancer.csv\")\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_enc, xtest_enc = prepare_inputs(xtrain, xtest)\n",
    "ytrain_enc, ytest_enc = prepare_targets(ytrain, ytest)\n",
    "\n",
    "model = LogisticRegression(solver='lbfgs')\n",
    "model.fit(xtrain_enc, ytrain_enc)\n",
    "\n",
    "yhat = model.predict(xtest_enc)\n",
    "\n",
    "accuracy = accuracy_score(ytest_enc, yhat)\n",
    "print('Accuracy: %.2f' % (accuracy*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8eec7f0",
   "metadata": {},
   "source": [
    "## Model built using Chi-Squared Features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2a27ead4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 74.74\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OrdinalEncoder, LabelEncoder\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    "def prepare_inputs(xtrain, xtest):\n",
    "    oe = OrdinalEncoder()\n",
    "    oe.fit(xtrain)\n",
    "    xtrain_enc = oe.transform(xtrain)\n",
    "    xtest_enc = oe.transform(xtest)\n",
    "    \n",
    "    return xtrain_enc, xtest_enc\n",
    "\n",
    "def prepare_targets(ytrain, ytest):\n",
    "    le = LabelEncoder()\n",
    "    le.fit(ytrain)\n",
    "    ytrain_enc = le.transform(ytrain)\n",
    "    ytest_enc = le.transform(ytest)\n",
    "    \n",
    "    return ytrain_enc, ytest_enc\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=chi2, k=4)\n",
    "    fs.fit(xtrain, ytrain)\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    \n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\BreastCancer.csv\")\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_enc, xtest_enc = prepare_inputs(xtrain, xtest)\n",
    "ytrain_enc, ytest_enc = prepare_targets(ytrain, ytest)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain_enc, ytrain_enc, xtest_enc)\n",
    "\n",
    "model = LogisticRegression(solver='lbfgs')\n",
    "model.fit(xtrain_fs, ytrain_enc)\n",
    "\n",
    "yhat = model.predict(xtest_fs)\n",
    "# evaluate predictions\n",
    "accuracy = accuracy_score(ytest_enc, yhat)\n",
    "print('Accuracy: %.2f' % (accuracy*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab577130",
   "metadata": {},
   "source": [
    "## Model built using Mutual Information Features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6d883bb7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 67.37\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OrdinalEncoder, LabelEncoder\n",
    "from sklearn.feature_selection import SelectKBest, chi2, mutual_info_classif\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    "def prepare_inputs(xtrain, xtest):\n",
    "    oe = OrdinalEncoder()\n",
    "    oe.fit(xtrain)\n",
    "    xtrain_enc = oe.transform(xtrain)\n",
    "    xtest_enc = oe.transform(xtest)\n",
    "    \n",
    "    return xtrain_enc, xtest_enc\n",
    "\n",
    "def prepare_targets(ytrain, ytest):\n",
    "    le = LabelEncoder()\n",
    "    le.fit(ytrain)\n",
    "    ytrain_enc = le.transform(ytrain)\n",
    "    ytest_enc = le.transform(ytest)\n",
    "    \n",
    "    return ytrain_enc, ytest_enc\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=mutual_info_classif, k=4)\n",
    "    fs.fit(xtrain, ytrain)\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    \n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\BreastCancer.csv\")\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_enc, xtest_enc = prepare_inputs(xtrain, xtest)\n",
    "ytrain_enc, ytest_enc = prepare_targets(ytrain, ytest)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain_enc, ytrain_enc, xtest_enc)\n",
    "\n",
    "model = LogisticRegression(solver='lbfgs')\n",
    "model.fit(xtrain_fs, ytrain_enc)\n",
    "\n",
    "yhat = model.predict(xtest_fs)\n",
    "\n",
    "accuracy = accuracy_score(ytest_enc, yhat)\n",
    "print('Accuracy: %.2f' % (accuracy*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d58a7b4d",
   "metadata": {},
   "source": [
    "# Numerical input data and a categorical (class) target variable (How to select Numerical Input Features)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea45131f",
   "metadata": {},
   "source": [
    "## ANOVA F-test Feature Selection:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "95a9bd6c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\talk2_000\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:63: FutureWarning: Arrays of bytes/strings is being converted to decimal numbers if dtype='numeric'. This behavior is deprecated in 0.24 and will be removed in 1.1 (renaming of 0.26). Please convert your data to numeric values explicitly instead.\n",
      "  return f(*args, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature 0: 16.527385\n",
      "Feature 1: 131.325562\n",
      "Feature 2: 0.042371\n",
      "Feature 3: 1.415216\n",
      "Feature 4: 12.778966\n",
      "Feature 5: 49.209523\n",
      "Feature 6: 13.377142\n",
      "Feature 7: 25.126440\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6UAAAHSCAYAAAAUmW0WAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAWrUlEQVR4nO3db4xlhXnf8d8T1nFiXMsgBkQBFSIht8QqtbWiSZEsK8Q1VizDG6RFsrVKXdFKjmu3RSmkL1BfWLJUFKUvmkgISLYKZUWxLaPITYw2sdy88J8Fm9oYE6jtwAbCTmqljlPJLs7TF3tsBlgCnjujZ+by+Uire8+558x9pKPVznfPuedWdwcAAAAm/MT0AAAAALx6iVIAAADGiFIAAADGiFIAAADGiFIAAADGiFIAAADGHJgeIEnOOeecvvjii6fHAAAAYBc88MADf9HdG6d7bU9E6cUXX5zjx49PjwEAAMAuqKo/fanXXL4LAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAmAPTA8BOuPzWQ9MjrI2Hbjw6PQIAAK8izpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAw5mWjtKrurKqTVfXVLev+Y1V9var+Z1V9oqreuOW1m6vq8ap6tKreuUtzAwAAsAZeyZnS30ly9QvW3Z/kzd39D5P8SZKbk6SqLktyKMnPLvv8ZlWdsWPTAgAAsFZeNkq7+7NJvv2CdZ/u7meXxc8luXB5fk2So939ve7+ZpLHk1yxg/MCAACwRnbiM6X/LMl/X55fkOTJLa+dWNa9SFXdUFXHq+r45ubmDowBAADAfrNSlFbVv0/ybJK7frjqNJv16fbt7tu6+2B3H9zY2FhlDAAAAPapA9vdsaoOJ3l3kqu6+4fheSLJRVs2uzDJU9sfDwAAgHW2rTOlVXV1kn+X5D3d/X+3vHRfkkNV9dqquiTJpUm+sPqYAAAArKOXPVNaVXcneXuSc6rqRJJbcupuu69Ncn9VJcnnuvtfdvfDVXVPkq/l1GW9H+juH+zW8AAAAOxvLxul3X39aVbf8bds/5EkH1llKAAAAF4dduLuuwAAALAtohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxLxulVXVnVZ2sqq9uWXd2Vd1fVY8tj2dtee3mqnq8qh6tqnfu1uAAAADsf6/kTOnvJLn6BetuSnKsuy9NcmxZTlVdluRQkp9d9vnNqjpjx6YFAABgrbxslHb3Z5N8+wWrr0lyZHl+JMm1W9Yf7e7vdfc3kzye5IqdGRUAAIB1s93PlJ7X3U8nyfJ47rL+giRPbtnuxLLuRarqhqo6XlXHNzc3tzkGAAAA+9lO3+ioTrOuT7dhd9/W3Qe7++DGxsYOjwEAAMB+sN0ofaaqzk+S5fHksv5Ekou2bHdhkqe2Px4AAADrbLtRel+Sw8vzw0k+uWX9oap6bVVdkuTSJF9YbUQAAADW1YGX26Cq7k7y9iTnVNWJJLck+WiSe6rq/UmeSHJdknT3w1V1T5KvJXk2yQe6+we7NDsAAAD73MtGaXdf/xIvXfUS238kyUdWGQoAAIBXh52+0REAAAC8YqIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMaIUAACAMStFaVX966p6uKq+WlV3V9VPVdXZVXV/VT22PJ61U8MCAACwXrYdpVV1QZJ/leRgd785yRlJDiW5Kcmx7r40ybFlGQAAAF5k1ct3DyT56ao6kOR1SZ5Kck2SI8vrR5Jcu+J7AAAAsKa2HaXd/WdJbk3yRJKnk/yf7v50kvO6++llm6eTnHu6/avqhqo6XlXHNzc3tzsGAAAA+9gql++elVNnRS9J8neTnFlV732l+3f3bd19sLsPbmxsbHcMAAAA9rFVLt/9xSTf7O7N7v5/ST6e5J8keaaqzk+S5fHk6mMCAACwjlaJ0ieS/FxVva6qKslVSR5Jcl+Sw8s2h5N8crURAQAAWFcHtrtjd3++qu5N8mCSZ5N8KcltSV6f5J6qen9Ohet1OzEoAAAA62fbUZok3X1LkltesPp7OXXWFAAAAP5Wq34lDAAAAGybKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGCMKAUAAGDMSlFaVW+sqnur6utV9UhV/XxVnV1V91fVY8vjWTs1LAAAAOtl1TOl/ynJ73f3309yeZJHktyU5Fh3X5rk2LIMAAAAL7LtKK2qNyR5W5I7kqS7v9/df5nkmiRHls2OJLl2tREBAABYV6ucKf2ZJJtJfruqvlRVt1fVmUnO6+6nk2R5PHcH5gQAAGANrRKlB5K8Nclvdfdbkvx1foxLdavqhqo6XlXHNzc3VxgDAACA/WqVKD2R5ER3f35ZvjenIvWZqjo/SZbHk6fbubtv6+6D3X1wY2NjhTEAAADYr7Ydpd3950merKo3LauuSvK1JPclObysO5zkkytNCAAAwNo6sOL+H0xyV1X9ZJJvJPnlnArde6rq/UmeSHLdiu8BAADAmlopSrv7y0kOnualq1b5uQAAALw6rPo9pQAAALBtohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxohQAAIAxB6YHAADYry6/9dD0CGvjoRuPTo8ADHGmFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDGiFAAAgDErR2lVnVFVX6qq31uWz66q+6vqseXxrNXHBAAAYB3txJnSDyV5ZMvyTUmOdfelSY4tywAAAPAiK0VpVV2Y5JeS3L5l9TVJjizPjyS5dpX3AAAAYH2teqb0N5L8apK/2bLuvO5+OkmWx3NXfA8AAADW1LajtKreneRkdz+wzf1vqKrjVXV8c3Nzu2MAAACwj61ypvTKJO+pqm8lOZrkF6rqd5M8U1XnJ8nyePJ0O3f3bd19sLsPbmxsrDAGAAAA+9W2o7S7b+7uC7v74iSHkvxhd783yX1JDi+bHU7yyZWnBAAAYC3txveUfjTJO6rqsSTvWJYBAADgRQ7sxA/p7s8k+czy/H8nuWonfi4AAADrbTfOlAIAAMArIkoBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYI0oBAAAYc2B6AAAA4NXn8lsPTY+wVh668ej0CNvmTCkAAABjRCkAAABjRCkAAABjRCkAAABjRCkAAABjRCkAAABjRCkAAABjRCkAAABjRCkAAABjRCkAAABjRCkAAABjth2lVXVRVf1RVT1SVQ9X1YeW9WdX1f1V9djyeNbOjQsAAMA6WeVM6bNJ/m13/4MkP5fkA1V1WZKbkhzr7kuTHFuWAQAA4EW2HaXd/XR3P7g8/6skjyS5IMk1SY4smx1Jcu2KMwIAALCmduQzpVV1cZK3JPl8kvO6++nkVLgmOXcn3gMAAID1s3KUVtXrk3wsyYe7+zs/xn43VNXxqjq+ubm56hgAAADsQytFaVW9JqeC9K7u/viy+pmqOn95/fwkJ0+3b3ff1t0Hu/vgxsbGKmMAAACwT61y991KckeSR7r717e8dF+Sw8vzw0k+uf3xAAAAWGcHVtj3yiTvS/KVqvrysu7Xknw0yT1V9f4kTyS5bqUJAQAAWFvbjtLu/uMk9RIvX7XdnwsAAMCrx47cfRcAAAC2Q5QCAAAwRpQCAAAwRpQCAAAwRpQCAAAwRpQCAAAwZpXvKX1VufzWQ9MjrI2Hbjw6PQIAALBHOFMKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAGFEKAADAmAPTAwAAwG64/NZD0yOsjYduPDo9AmtMlALAHuaX6p3jl2qAvcnluwAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIwRpQAAAIw5MD0AALMuv/XQ9Ahr46Ebj06PAAD7zq6dKa2qq6vq0ap6vKpu2q33AQAAYP/alSitqjOS/Ock70pyWZLrq+qy3XgvAAAA9q/dOlN6RZLHu/sb3f39JEeTXLNL7wUAAMA+tVtRekGSJ7csn1jWAQAAwI9Ud+/8D626Lsk7u/ufL8vvS3JFd39wyzY3JLlhWXxTkkd3fJBXp3OS/MX0ELwkx2dvc3z2Nsdnb3N89jbHZ29zfPY2x2dn/L3u3jjdC7t1990TSS7asnxhkqe2btDdtyW5bZfe/1Wrqo5398HpOTg9x2dvc3z2Nsdnb3N89jbHZ29zfPY2x2f37dblu19McmlVXVJVP5nkUJL7dum9AAAA2Kd25Uxpdz9bVb+S5A+SnJHkzu5+eDfeCwAAgP1rty7fTXd/Ksmnduvn85JcEr23OT57m+Oztzk+e5vjs7c5Pnub47O3OT67bFdudAQAAACvxG59phQAAABelihdE1V1dVU9WlWPV9VN0/PwfFV1Z1WdrKqvTs/C81XVRVX1R1X1SFU9XFUfmp6J51TVT1XVF6rqoeX4/IfpmXixqjqjqr5UVb83PQvPV1XfqqqvVNWXq+r49Dw8X1W9saruraqvL/8O/fz0TDynqt60/N354Z/vVNWHp+daRy7fXQNVdUaSP0nyjpz6Op4vJrm+u782Ohg/UlVvS/LdJP+lu988PQ/Pqarzk5zf3Q9W1d9J8kCSa/392RuqqpKc2d3frarXJPnjJB/q7s8Nj8YWVfVvkhxM8obufvf0PDynqr6V5GB3+47FPaiqjiT5H919+/KNFa/r7r8cHovTWH7f/rMk/7i7/3R6nnXjTOl6uCLJ4939je7+fpKjSa4ZnoktuvuzSb49PQcv1t1Pd/eDy/O/SvJIkgtmp+KH+pTvLouvWf7439Q9pKouTPJLSW6fngX2k6p6Q5K3JbkjSbr7+4J0T7sqyf8SpLtDlK6HC5I8uWX5RPxSDT+2qro4yVuSfH54FLZYLg39cpKTSe7vbsdnb/mNJL+a5G+G5+D0Osmnq+qBqrphehie52eSbCb57eXy99ur6szpoXhJh5LcPT3EuhKl66FOs86ZBPgxVNXrk3wsyYe7+zvT8/Cc7v5Bd/+jJBcmuaKqXAK/R1TVu5Oc7O4HpmfhJV3Z3W9N8q4kH1g+TsLecCDJW5P8Vne/JclfJ3FfkD1oubT6PUn+2/Qs60qUrocTSS7asnxhkqeGZoF9Z/ms4seS3NXdH5+eh9NbLmv7TJKrZydhiyuTvGf53OLRJL9QVb87OxJbdfdTy+PJJJ/IqY/8sDecSHJiy9Uf9+ZUpLL3vCvJg939zPQg60qUrocvJrm0qi5Z/ifnUJL7hmeCfWG5kc4dSR7p7l+fnofnq6qNqnrj8vynk/xikq+PDsWPdPfN3X1hd1+cU//2/GF3v3d4LBZVdeZyA7csl4X+0yTuAr9HdPefJ3myqt60rLoqiZvs7U3Xx6W7u+rA9ACsrrufrapfSfIHSc5Icmd3Pzw8FltU1d1J3p7knKo6keSW7r5jdioWVyZ5X5KvLJ9bTJJf6+5PzY3EFucnObLc9fAnktzT3b52BF6Z85J84tT/veVAkv/a3b8/OxIv8MEkdy0nFb6R5JeH5+EFqup1OfUNF/9iepZ15ithAAAAGOPyXQAAAMaIUgAAAMaIUgAAAMaIUgAAAMaIUgAAAMaIUgAAAMaIUgAAAMaIUgAAAMb8fytfrmEavt+jAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 1152x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OrdinalEncoder, LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=f_classif, k='all')\n",
    "\n",
    "    fs.fit(xtrain, ytrain)\n",
    "\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\pima-indians-diabetes.csv\")\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain, ytrain, xtest)\n",
    "\n",
    "for i in range(len(fs.scores_)):\n",
    "    print('Feature %d: %f' % (i, fs.scores_[i]))\n",
    "\n",
    "plt.figure(figsize=(16, 8))\n",
    "plt.bar([i for i in range(len(fs.scores_))], fs.scores_, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "905af674",
   "metadata": {},
   "source": [
    "## Mutual Information Feature Selection:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a45b0e38",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\talk2_000\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:63: FutureWarning: Arrays of bytes/strings is being converted to decimal numbers if dtype='numeric'. This behavior is deprecated in 0.24 and will be removed in 1.1 (renaming of 0.26). Please convert your data to numeric values explicitly instead.\n",
      "  return f(*args, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature 0: 0.015048\n",
      "Feature 1: 0.094899\n",
      "Feature 2: 0.038365\n",
      "Feature 3: 0.001671\n",
      "Feature 4: 0.041853\n",
      "Feature 5: 0.066062\n",
      "Feature 6: 0.010857\n",
      "Feature 7: 0.056153\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHSCAYAAAAezFYoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAUWUlEQVR4nO3dYahndX7f8c83MyvJmi4WHIpV6bgg20rArAzWVFhKNm0ds6x9UhhhVxCKFXS721aCyZOlj/pEQiqIImqIZLtD625gSIdsSpMl3QduHd3VjZk1TO2mTjR1QombjQ+szbcP7r/05uvV+/d6r+defb3g4v2f8ztzv/+DDG+P5/5PdXcAAID/78eWHgAAAPYbkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwHF56gK1ceumlffTo0aXHAADgA+zpp5/+0+4+stW+fRnJR48ezZkzZ5YeAwCAD7Cq+qO32+d2CwAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAIbDSw/AwXftfSeWHmERz95zcukRAIA94koyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBhrUiuqpuq6oWqOldV926xv6rq/tX+56rquk37/kVVPV9Vv19VX62qH9/NNwAAALtt20iuqkNJHkhyPMk1SW6tqmvGsuNJrl593ZHkwdWxlyf550mOdfdPJTmU5MSuTQ8AAHtgnSvJ1yc5190vdvcbSU4muWWsuSXJ473hySSXVNVlq32Hk/xEVR1O8tEkL+/S7AAAsCfWieTLk7y06fX51bZt13T3Hye5L8n/SPJKkte6+7d3Pi4AAOy9dSK5ttjW66ypqr+ejavMVyX5m0kurqrPbflDqu6oqjNVdebChQtrjAUAAHtjnUg+n+TKTa+vyFtvmXi7NT+X5L9394Xu/t9Jvp7k7231Q7r74e4+1t3Hjhw5su78AACw69aJ5KeSXF1VV1XVRdn4xbtTY82pJLetPuXihmzcVvFKNm6zuKGqPlpVleTTSc7u4vwAALDrDm+3oLvfrKq7k3wjG59O8Vh3P19Vd672P5TkdJKbk5xL8nqS21f7vl1VTyR5JsmbSb6T5OG9eCMAALBbto3kJOnu09kI4c3bHtr0fSe5622O/XKSL7+HGQEA4H3liXsAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwHB46QEAYL+69r4TS4+wmGfvObn0CLAoV5IBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAADDWpFcVTdV1QtVda6q7t1if1XV/av9z1XVdZv2XVJVT1TV96vqbFX9zG6+AQAA2G3bRnJVHUryQJLjSa5JcmtVXTOWHU9y9errjiQPbtr3b5P8Vnf/7STXJjm7C3MDAMCeWedK8vVJznX3i939RpKTSW4Za25J8nhveDLJJVV1WVV9LMmnkjyaJN39Rnf/2e6NDwAAu2+dSL48yUubXp9fbVtnzceTXEjyq1X1nap6pKoufg/zAgDAnlsnkmuLbb3mmsNJrkvyYHd/MslfJHnLPc1JUlV3VNWZqjpz4cKFNcYCAIC9sU4kn09y5abXVyR5ec0155Oc7+5vr7Y/kY1ofovufri7j3X3sSNHjqwzOwAA7Il1IvmpJFdX1VVVdVGSE0lOjTWnkty2+pSLG5K81t2vdPefJHmpqj6xWvfpJH+wW8MDAMBeOLzdgu5+s6ruTvKNJIeSPNbdz1fVnav9DyU5neTmJOeSvJ7k9k1/xBeSfGUV2C+OfQAAsO9sG8lJ0t2nsxHCm7c9tOn7TnLX2xz73STHdj4iAAC8vzxxDwAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAcHjpAQAASK6978TSIyzm2XtOLj3CW7iSDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADAcXnoAAPbetfedWHqExTx7z8mlRwAOIFeSAQBgEMkAADCIZAAAGEQyAAAMa0VyVd1UVS9U1bmquneL/VVV96/2P1dV1439h6rqO1X1m7s1OAAA7JVtI7mqDiV5IMnxJNckubWqrhnLjie5evV1R5IHx/4vJjn7nqcFAID3wTpXkq9Pcq67X+zuN5KcTHLLWHNLksd7w5NJLqmqy5Kkqq5I8vNJHtnFuQEAYM+sE8mXJ3lp0+vzq23rrvmVJL+Q5C/f6YdU1R1Vdaaqzly4cGGNsQAAYG+sE8m1xbZeZ01VfSbJq9399HY/pLsf7u5j3X3syJEja4wFAAB7Y51IPp/kyk2vr0jy8pprbkzy2ar6QTZu0/jZqvr1HU8LAADvg3Ui+akkV1fVVVV1UZITSU6NNaeS3Lb6lIsbkrzW3a909y929xXdfXR13O909+d28w0AAMBuO7zdgu5+s6ruTvKNJIeSPNbdz1fVnav9DyU5neTmJOeSvJ7k9r0bGQAA9ta2kZwk3X06GyG8edtDm77vJHdt82d8M8k33/WEAADwPvPEPQAAGNa6kgzsvmvvO7H0CIt59p6TS48AAO/IlWQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAMNakVxVN1XVC1V1rqru3WJ/VdX9q/3PVdV1q+1XVtXvVtXZqnq+qr64228AAAB227aRXFWHkjyQ5HiSa5LcWlXXjGXHk1y9+rojyYOr7W8m+Vfd/XeS3JDkri2OBQCAfWWdK8nXJznX3S929xtJTia5Zay5JcnjveHJJJdU1WXd/Up3P5Mk3f3nSc4muXwX5wcAgF23TiRfnuSlTa/P562hu+2aqjqa5JNJvv2upwQAgPfROpFcW2zrd7Omqn4yydeSfKm7f7jlD6m6o6rOVNWZCxcurDEWAADsjXUi+XySKze9viLJy+uuqaqPZCOQv9LdX3+7H9LdD3f3se4+duTIkXVmBwCAPbFOJD+V5OqquqqqLkpyIsmpseZUkttWn3JxQ5LXuvuVqqokjyY5292/vKuTAwDAHjm83YLufrOq7k7yjSSHkjzW3c9X1Z2r/Q8lOZ3k5iTnkrye5PbV4Tcm+XyS71XVd1fbfqm7T+/quwAAgF20bSQnySpqT49tD236vpPctcVx38rW9ysDAMC+5Yl7AAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYDi89AD7ybX3nVh6hEU8e8/JpUcAANhXXEkGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIDh8NIDAAAfLNfed2LpERbz7D0nlx6BXeJKMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABgOLz0AwLtx7X0nlh5hMc/ec3LpEQA+NFxJBgCAQSQDAMCwViRX1U1V9UJVnauqe7fYX1V1/2r/c1V13brHAgDAfrNtJFfVoSQPJDme5Jokt1bVNWPZ8SRXr77uSPLguzgWAAD2lXWuJF+f5Fx3v9jdbyQ5meSWseaWJI/3hieTXFJVl615LAAA7CvrRPLlSV7a9Pr8ats6a9Y5FgAA9pXq7ndeUPVPkvyj7v6nq9efT3J9d39h05r/mOTfdPe3Vq//c5JfSPLx7Y7d9GfckY1bNZLkE0leeI/v7aC5NMmfLj3EAeS87YzztjPO2844bzvjvO2M87ZzH8Zz97e6+8hWO9b5nOTzSa7c9PqKJC+vueaiNY5NknT3w0keXmOeD6SqOtPdx5ae46Bx3nbGedsZ521nnLedcd52xnnbOefur1rndounklxdVVdV1UVJTiQ5NdacSnLb6lMubkjyWne/suaxAACwr2x7Jbm736yqu5N8I8mhJI919/NVdedq/0NJTie5Ocm5JK8nuf2djt2TdwIAALtkrcdSd/fpbITw5m0Pbfq+k9y17rFs6UN7q8l75LztjPO2M87bzjhvO+O87YzztnPO3Sbb/uIeAAB82HgsNQAADCJ5H/Do7nevqh6rqler6veXnuUgqaorq+p3q+psVT1fVV9ceqaDoKp+vKr+a1U9uzpv/3rpmQ6SqjpUVd+pqt9cepaDoqp+UFXfq6rvVtWZpec5KKrqkqp6oqq+v/p77meWnmm/q6pPrP49+39fP6yqLy09137gdouFrR7d/YdJ/kE2PkrvqSS3dvcfLDrYPldVn0ryo2w86fGnlp7noFg9CfOy7n6mqv5akqeT/GP/vr2zqqokF3f3j6rqI0m+leSLqyeMso2q+pdJjiX5WHd/Zul5DoKq+kGSY939YfvM2vekqn4tyX/p7kdWn6r10e7+s4XHOjBWTfLHSf5ud//R0vMszZXk5Xl09w509+8l+V9Lz3HQdPcr3f3M6vs/T3I2noK5rd7wo9XLj6y+XGFYQ1VdkeTnkzyy9Cx8sFXVx5J8KsmjSdLdbwjkd+3TSf6bQN4gkpfn0d0soqqOJvlkkm8vPMqBsLpl4LtJXk3yn7rbeVvPr2TjCax/ufAcB00n+e2qenr1RFq29/EkF5L86ur2nkeq6uKlhzpgTiT56tJD7BcieXm1xTZXqNhTVfWTSb6W5Evd/cOl5zkIuvv/dPdPZ+PJoddXldt8tlFVn0nyanc/vfQsB9CN3X1dkuNJ7lrdYsY7O5zkuiQPdvcnk/xFEr/ns6bV7SmfTfIflp5lvxDJy1vnsd+wa1b31H4tyVe6++tLz3PQrP737TeT3LTsJAfCjUk+u7q/9mSSn62qX192pIOhu19e/fPVJL+RjVvzeGfnk5zf9H95nshGNLOe40me6e7/ufQg+4VIXp5Hd/O+Wf0C2qNJznb3Ly89z0FRVUeq6pLV9z+R5OeSfH/RoQ6A7v7F7r6iu49m4++23+nuzy081r5XVRevfrE2q9sF/mESn+Szje7+kyQvVdUnVps+ncQvJa/v1rjV4q9Y64l77B2P7t6Zqvpqkr+f5NKqOp/ky9396LJTHQg3Jvl8ku+t7q9Nkl9aPRmTt3dZkl9b/eb3jyX5993t48zYK38jyW9s/DdtDif5d939W8uOdGB8IclXVhedXkxy+8LzHAhV9dFsfMrWP1t6lv3ER8ABAMDgdgsAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAw/F9RgQ19yFRhPQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import OrdinalEncoder, LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=mutual_info_classif, k='all')\n",
    "\n",
    "    fs.fit(xtrain, ytrain)\n",
    "\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\pima-indians-diabetes.csv\")\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain, ytrain, xtest)\n",
    "\n",
    "for i in range(len(fs.scores_)):\n",
    "    print('Feature %d: %f' % (i, fs.scores_[i]))\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.bar([i for i in range(len(fs.scores_))], fs.scores_, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a25f780c",
   "metadata": {},
   "source": [
    "## Model built using all Features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a5d60c5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 77.56\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\talk2_000\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:63: FutureWarning: Arrays of bytes/strings is being converted to decimal numbers if dtype='numeric'. This behavior is deprecated in 0.24 and will be removed in 1.1 (renaming of 0.26). Please convert your data to numeric values explicitly instead.\n",
      "  return f(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    "\n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\pima-indians-diabetes.csv\")\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "model = LogisticRegression(solver='liblinear')\n",
    "model.fit(xtrain, ytrain)\n",
    "\n",
    "yhat = model.predict(xtest)\n",
    "\n",
    "accuracy = accuracy_score(ytest, yhat)\n",
    "print('Accuracy: %.2f' % (accuracy*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5a2497f",
   "metadata": {},
   "source": [
    "## Model built using ANOVA F-test Features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "38f8f809",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 78.74\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\talk2_000\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:63: FutureWarning: Arrays of bytes/strings is being converted to decimal numbers if dtype='numeric'. This behavior is deprecated in 0.24 and will be removed in 1.1 (renaming of 0.26). Please convert your data to numeric values explicitly instead.\n",
      "  return f(*args, **kwargs)\n",
      "C:\\Users\\talk2_000\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:63: FutureWarning: Arrays of bytes/strings is being converted to decimal numbers if dtype='numeric'. This behavior is deprecated in 0.24 and will be removed in 1.1 (renaming of 0.26). Please convert your data to numeric values explicitly instead.\n",
      "  return f(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=f_classif, k=4)\n",
    "\n",
    "    fs.fit(xtrain, ytrain)\n",
    "\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\pima-indians-diabetes.csv\")\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain, ytrain, xtest)\n",
    "\n",
    "model = LogisticRegression(solver='liblinear')\n",
    "model.fit(xtrain_fs, ytrain)\n",
    "\n",
    "yhat = model.predict(xtest_fs)\n",
    "\n",
    "accuracy = accuracy_score(ytest, yhat)\n",
    "print('Accuracy: %.2f' % (accuracy*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bcb07455",
   "metadata": {},
   "source": [
    "## Model Built Using Mutual Information Features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1a8696b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 77.56\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\talk2_000\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:63: FutureWarning: Arrays of bytes/strings is being converted to decimal numbers if dtype='numeric'. This behavior is deprecated in 0.24 and will be removed in 1.1 (renaming of 0.26). Please convert your data to numeric values explicitly instead.\n",
      "  return f(*args, **kwargs)\n",
      "C:\\Users\\talk2_000\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:63: FutureWarning: Arrays of bytes/strings is being converted to decimal numbers if dtype='numeric'. This behavior is deprecated in 0.24 and will be removed in 1.1 (renaming of 0.26). Please convert your data to numeric values explicitly instead.\n",
      "  return f(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=mutual_info_classif, k=4)\n",
    "\n",
    "    fs.fit(xtrain, ytrain)\n",
    "\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\pima-indians-diabetes.csv\")\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain, ytrain, xtest)\n",
    "\n",
    "model = LogisticRegression(solver='liblinear')\n",
    "model.fit(xtrain_fs, ytrain)\n",
    "\n",
    "yhat = model.predict(xtest_fs)\n",
    "\n",
    "accuracy = accuracy_score(ytest, yhat)\n",
    "print('Accuracy: %.2f' % (accuracy*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93e777a3",
   "metadata": {},
   "source": [
    "## Tune the number of selected Features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "678c0622",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Mean Accuracy: 0.770\n",
      "Best Config: {'anova__k': 7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\talk2_000\\anaconda3\\lib\\site-packages\\sklearn\\utils\\validation.py:63: FutureWarning: Arrays of bytes/strings is being converted to decimal numbers if dtype='numeric'. This behavior is deprecated in 0.24 and will be removed in 1.1 (renaming of 0.26). Please convert your data to numeric values explicitly instead.\n",
      "  return f(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\pima-indians-diabetes.csv\")\n",
    "\n",
    "cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "\n",
    "model = LogisticRegression(solver='liblinear')\n",
    "fs = SelectKBest(score_func=f_classif)\n",
    "pipeline = Pipeline(steps=[('anova',fs), ('lr', model)])\n",
    "\n",
    "grid = dict()\n",
    "grid['anova__k'] = [i+1 for i in range(x.shape[1])]\n",
    "\n",
    "search = GridSearchCV(pipeline, grid, scoring='accuracy', n_jobs=-1, cv=cv)\n",
    "\n",
    "results = search.fit(x, y)\n",
    "\n",
    "print('Best Mean Accuracy: %.3f' % results.best_score_)\n",
    "print('Best Config: %s' % results.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "0e7efb3b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">1 0.748 (0.048)\n",
      ">2 0.756 (0.042)\n",
      ">3 0.761 (0.044)\n",
      ">4 0.759 (0.042)\n",
      ">5 0.770 (0.041)\n",
      ">6 0.766 (0.042)\n",
      ">7 0.770 (0.042)\n",
      ">8 0.768 (0.040)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA6gAAAHSCAYAAADhZ+amAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAgA0lEQVR4nO3df4zfd30f8OeLsyGQktRWPFQIXajE6FWWStmJspHReQyabAXWVqrwtE2gk6JIYNFO6kp1SFCh2z+tpkmFzUIc66bRQy0Qkq5TSKW57a4qa85ZQhLcbB6l4KZrLrLXrtDAYb/3hy/e4Vxy33Puvp/393uPh3Syv58f/rze37e/53v6836/P9VaCwAAAAztBUMXAAAAAImACgAAQCcEVAAAALogoAIAANAFARUAAIAuCKgAAAB04cDQBWzlpptuarfccsvQZQAAALDLTp8+/WRr7chW+7oMqLfccktWV1eHLgMAAIBdVlV//Gz7DPEFAACgCwIqAAAAXRBQAQAA6MJIAbWqbquqx6rqbFW9f4v9N1bVb1TVQ1X1aFW9e9O+r1TVw1X1YFWZWAoAAMCWtl0kqapmknw0yVuSnEtyf1Xd01r70qbD3pPkS621t1XVkSSPVdUnW2vf2th/rLX25G4XDwAAwPQY5Q7q65Ocba19eSNwfirJO646piV5aVVVku9Kcj7Jt3e1UgAAAKbaKAH1FUm+tun1uY1tm30kyWySx5M8nOR9rbVLG/takvuq6nRV3fE86wUAAGBKjRJQa4tt7arXP5rkwSQvT/LaJB+pqhs29r2xtfa6JLcneU9VvWnLi1TdUVWrVbW6trY2Su0AAABMkVEC6rkkr9z0+uZcvlO62buTfLZddjbJHyX5/iRprT2+8esTSe7K5SHDz9Ba+1hrba61NnfkyJGdtQIAAICJN0pAvT/Jq6vqVVX1wiTvTHLPVcd8Ncmbk6SqXpbkNUm+XFXXV9VLN7Zfn+StSR7ZreIBAACYHtuu4tta+3ZVvTfJ55PMJPlEa+3RqrpzY//JJB9O8itV9XAuDwn+udbak1X1fUnuurx2Ug4k+dXW2r171BYAAAAmWLV29XTS4c3NzbXVVY9MBQAAmDZVdbq1NrfVvlGG+AIAAMCeE1ABAADogoAKAABAFwRUAAAAuiCgAgAA0AUBFQAAgC5s+xxUgN5sPFt5rHp8JBcA+4N/9yab/tsZARWYONf6TbeqJvobNgD7k3/3Jpv+2xlDfAEAAOiCgAoAAEAXBFQAAAC6IKACAADQBQEVAACALgioAAAAdEFABQAAoAsCKgAAAF0QUAEAAOiCgAoAAEAXBFQAAAC6IKACAADQBQEVAACALgioAAAAdEFABQAAoAsCKgAAAF0QUAEAAOiCgAoAAEAXBFQAAAC6IKACAADQBQEVAACALgioAAAAdOHA0AVMqqoa+zVba2O/5rTSfzAMn73Jpv8ml74DJoWAeo2u9ZtuVfmG3QH9B8Pw2Zts+m9y6TtgUhjiCwAAQBcEVAAAALogoAIAANAFARUAAIAuCKgAAAB0QUAFAACgCwIqAAAAXRBQAQAA6IKACgAAQBcEVAAAALogoAIAANAFARUAAIAuCKgAAAB0QUAFAACgCwIqAAAAXRBQAQAA6IKACgAAQBcEVAAAALogoAIAANAFARUAAIAuCKgAAAB0QUAFAACgCwIqAAAAXRBQAQAA6IKACgAAQBcEVAAAALogoAIAANAFARUAAIAuCKgAAAB0QUAFAACgCwIqAAAAXRBQAQAA6IKACgAAQBcEVAAAALogoAIAANAFARUAAIAuCKgAAAB0QUAFAACgCwIqAAAAXRBQAQAA6IKACgAAQBcEVAAAALogoAIAANAFARUAAIAuCKgAAAB0QUAFAACgCwIqAAAAXTgwdAEAAAC9O3z4cC5cuDDWa1bVWK5z6NChnD9/fizX2o6ACgAAsI0LFy6ktTZ0GXtiXEF4FIb4AgAA0AUBFQAAgC4IqAAAAHRhpIBaVbdV1WNVdbaq3r/F/hur6jeq6qGqerSq3j3quQAAsB8cPnw4VTW2ryRjvd7hw4cHfoeZBtsuklRVM0k+muQtSc4lub+q7mmtfWnTYe9J8qXW2tuq6kiSx6rqk0kujnAuAABMvWleZCfpa6EdJtcod1Bfn+Rsa+3LrbVvJflUkndcdUxL8tK6/Lfyu5KcT/LtEc8FAACAkQLqK5J8bdPrcxvbNvtIktkkjyd5OMn7WmuXRjwXAAAARgqoW92rv3pswo8meTDJy5O8NslHquqGEc+9fJGqO6pqtapW19bWRigLAACAaTJKQD2X5JWbXt+cy3dKN3t3ks+2y84m+aMk3z/iuUmS1trHWmtzrbW5I0eOjFo/AAAAU2KUgHp/kldX1auq6oVJ3pnknquO+WqSNydJVb0syWuSfHnEcwEAAGD7VXxba9+uqvcm+XySmSSfaK09WlV3buw/meTDSX6lqh7O5WG9P9daezJJtjp3b5oCAADAJKsel7qem5trq6urQ5exJ6pqqpcXn3b6b7Lpv8ml7yab/ptc+m53Tfv7qX2Ta9xtq6rTrbW5rfaNMsQXAAAA9pyACgAAQBcEVAAAALogoAIAANAFARUAAIAuCKgAAAB0QUAFAACgCwIqAAAAXRBQAQAA6MKBoQsAYPIcPnw4Fy5cGOs1q2ps1zp06FDOnz8/tuuNm/6bXPoOmHYCKgA7duHChbTWhi5jz4zzB/Ih6L/Jpe+AaWeILwAAAF0QUAEAAOiCgAoAAEAXBFQAAAC6sO8XSbIaHgAAQB/2fUC1Gh4AAEAfDPEFAACgCwIqAAAAXdj3Q3yB4UzzHHDzvwEAdk5ABQYzzXPAzf8GANg5Q3wBAADogoAKAABAFwRUAAAAuiCgAgAA0AUBFQAAgC4IqAAAAHRBQAUAAKALAioAAABdEFABAADogoAKAABAFwRUAAAAuiCgAgAA0IUDQxcAz8fhw4dz4cKFsV6zqsZ2rUOHDuX8+fNjux4AAAxJQGWiXbhwIa21ocvYM+MMwwAAMDRDfAEAAOiCgAoAAEAXBFQAAAC6IKACAADQBQEVAACALgioAAAAdEFABQAAoAsCKgAAAF0QUAEAAOiCgAoAAEAXBFQAAAC6IKACAADQBQEVAACALhwYugAAYLzaB29IPnTjWK61NvOC/OyRm/JLa0/mpouXxnLN9sEbxnId2CmfPdhetdaGruEZ5ubm2urq6liuVVXp8T3YLdo32bRvck1z2xLtm3TjbN+Hv/Dh/Ppjv56fes1P5QNv+MBYrjnN/TfNbUu0bzf57O2+aW7fuNtWVadba3Nb7TPEFwDYE2vfWMvdZ+9OS8vnzn4uT/7Vk0OXBPuCzx6TTEAFAPbEyS+ezKV2eWjhpXYpJx86OXBFsD/47E2+tW+s5V33vmtf/ueCgAoA7Lqn7+CsX1pPkqxfWncnB8bAZ286nPziyTzwZw/sy/9cEFABgF23+Q7O09zJgb3nszf59vsQbQEVANh1Dz3x0JU7OE9bv7SeB594cJiCYJ/w2Zt8+32ItlV8p3g1rkT7Jp32Ta5pbluifZNO+ybXNLct0b5Jp33P39o31nL7Z2/PNy9+88q2F828KPf+5L256cU37dl1reILAEDX9vMiLTAUQ7QFVAAAtrCfF2mBoRiibYivoQgTTvsm2zS3b5rblmjfpNO+CfahG8dymbWZF+T2m1+eb77gBXnRpUu599zjuenipe1P3A0f+vPxXGcAU/13M9PfvnF9/gYzxs/ecw3xFVCn/IOkfZNN+ybXNLct0b5Jp32Ta1xt+/AXPpy7/uddWb+0noMvOJifePVP5ANv+MCeX3ea+y7Rvkk3ze0zBxUAgC55jiYwJAEVAIArLNICDElABQDgCou0AEM6MHQBAAD049Nv//TQJQD7mDuoAAAAdEFABfYFD5wHAOifgArsCx44DwDQPwEVmHpPPzKhpXlUAgBAxwRUYOptfmSCRyUAAPRLQAWmmgfOTz7zhwFg/xBQganmgfOTz/xhANg/BFRgqnng/GQzfxgA9pcDQxcAsJc8cH6ybTV/+ANv+MDAVQEAe8UdVAC6ZP4wAOw/AiqMyEItMF7mDwPA/iOgwogs1ALjZf4wAOw/5qDCCK5eqOXOH7wzN734pqHLgqlm/jAA7D/uoMIItlqoBQAA2F0CKmzDQi0AADAeAipsw0ItAAAwHgIqbMNCLQAAMB4WSYJtWKgFAADGQ0BlorUP3pB86Mahy9gz7YM3DF3Cnprm/tN3k23a+y9JqmroEvbMoUOHhi4BgGtUrbWha3iGubm5trq6OpZrVVV6fA92i/ZNNu2bXNPctkT72Bnv5+6Z9vdS+yab9k2ucbetqk631ua22jfSHNSquq2qHquqs1X1/i32/2xVPbjx9UhVXayqwxv7vlJVD2/sG0/qBAAAYOJsO8S3qmaSfDTJW5KcS3J/Vd3TWvvS08e01n4xyS9uHP+2JD/TWju/6Y851lrzTA4AAACe1Sh3UF+f5Gxr7cuttW8l+VSSdzzH8ceTLO9GcQAAAOwfowTUVyT52qbX5za2PUNVvSTJbUk+s2lzS3JfVZ2uqjuutVAAAACm2ygBdatl/p5tBu3bkvzeVcN739hae12S25O8p6retOVFqu6oqtWqWl1bWxuhrMmz9o21vOved+XJvzLaGQAA4GqjBNRzSV656fXNSR5/lmPfmauG97bWHt/49Ykkd+XykOFnaK19rLU211qbO3LkyAhlTZ6TXzyZB/7sgZx86OTQpQAAAHRnlIB6f5JXV9WrquqFuRxC77n6oKq6McmPJLl707brq+qlT/8+yVuTPLIbhU+atW+s5e6zd6el5XNnP+cuKgAAwFW2DaittW8neW+Szyc5k+TXWmuPVtWdVXXnpkN/PMl9rbWvb9r2siQrVfVQkj9I8puttXt3r/zJcfKLJ3OpXUqSXGqX3EUFAAC4SvX4sNm5ubm2ujqeR6aO46G0a99Yy+2fvT3fvPjNK9teNPOi3PuT9+amF9+0p9ee5gcKJ9o36aa5fdPctkT72Bnv5+6Z9vdS+yab9k2ucbetqk631ua22jfKEF+ep813T5/mLioAAMB3ElDH4KEnHsr6pfXv2LZ+aT0PPvHgMAUBAAB06MDQBewHn377p4cuAQAAoHvuoAIAANCFfX8HtX3whuRDNw5dxp5pH7xh6BIAAABGsu8Dav3CX0ztalzJxopcHxq6CgAAgO0Z4gsAAEAX9v0dVAAAgFFU1dAl7IlDhw4NXcIVAioAAMA2xj0tsKqmeiriszHEFwAAgC4IqAAAAHRBQAUAAKALAioAAABdsEgSE29aV1NL+lpRba9Ma//pu8m2H/qPyeWzN9n0Hzw3AZWJZjW1yab/Jpe+g2H47E02/QfbM8QXAACALgioAAAAdEFABQAAoAsCKgAAAF0QUAEAAOiCgAoAAEAXBFQAAAC6IKACAADQBQEVAACALgioAAAAdEFABQAAoAsCKgAAAF0QUAEAAOiCgAoAAEAXBFQAAAC6IKACAADQBQEVAACALgioAAAAdEFABQAAoAsCKgAAAF0QUAEAAOiCgAoAAEAXBFQAAAC6IKACAADQBQEVAACALgioAAAAdEFABQAAoAsCKgAAAF0QUAEAAOiCgAoAAEAXBFQAAAC6IKACAADQBQEVAACALgioAAAAdEFABQAAoAsCKgAAAF0QUAEAAOiCgAoAAEAXBFQAAAC6IKACAADQBQEVAACALgioAAAAdEFABQAAoAsCKgAAAF0QUAEAAOiCgAoAAEAXDgxdAAAwGapq7Oe21q75mgBMHgEVABiJsAjAXjPEFwAAgC4IqAAAAHRBQAUAAKALAioAAABdEFABAADogoAKAABAFwRUAAAAuiCgAgAA0AUBFQAAgC4IqAAAAHRBQAUAAKALAioAAABdODB0AT2oqqFL2DOHDh0augQAAICR7PuA2lob6/WqauzXBAAAmASG+AIAANAFARUAAIAuCKgAAAB0QUAFAACgCwIqAAAAXRBQAQAA6IKACgAAQBdGCqhVdVtVPVZVZ6vq/Vvs/9mqenDj65GqulhVh0c5FwAAAJIRAmpVzST5aJLbk/xAkuNV9QObj2mt/WJr7bWttdcm+fkkv9NaOz/KuQAAAJCMdgf19UnOtta+3Fr7VpJPJXnHcxx/PMnyNZ4LAADAPjVKQH1Fkq9ten1uY9szVNVLktyW5DM7PRcAAID9bZSAWltsa89y7NuS/F5r7fxOz62qO6pqtapW19bWRigLAACAaTJKQD2X5JWbXt+c5PFnOfad+f/De3d0bmvtY621udba3JEjR0YoCwAAgGkySkC9P8mrq+pVVfXCXA6h91x9UFXdmORHkty903MBAADgwHYHtNa+XVXvTfL5JDNJPtFae7Sq7tzYf3Lj0B9Pcl9r7evbnbvbjQAAAGDyVWvPNp10OHNzc211dXXoMvZEVaXH95zR6L/Jpv8ml76DYfjsTTb9N9mmuf+q6nRrbW6rfdveQYVpVLXV+l17e+60foMBAIDdIqCyLwmLAADQn1EWSQIAAIA9J6ACAADQBQEVAACALgioAAAAdEFABQAAoAsCKgAAAF0QUAEAAOiCgAoAAEAXBFQAAAC6IKACAADQBQEVAACALgioAAAAdEFABQAAoAsCKgAAAF0QUAEAAOjCgaELAGD/qKqxn9tau+ZrAgDjJaACMDbCIgDwXAzxBQAAoAsCKgAAAF0QUAEAAOiCgAoAAEAXBFQAAAC6YBVfYOJ4VAkAwHQSUIGJIywCAEwnQ3wBAADogoAKAABAFwRUAAAAuiCgAgAA0AUBFQAAgC4IqAAAAHRBQAUAAKALAioAAABdEFABAADogoAKAABAFwRUAAAAuiCgAgAA0AUBFQAAgC4IqAAAAHRBQAUAAKALB4YuAACAvVVVYz+3tXbN1wT2LwEVAGDKCYvApDDEFwAAgC4IqAAAAHRBQAUAAKALAioAsCeWl5dz9OjRzMzM5OjRo1leXh66JAA6Z5EkAGDXLS8vZ2FhIUtLS7n11luzsrKS+fn5JMnx48cHrg6AXrmDCgDsusXFxSwtLeXYsWM5ePBgjh07lqWlpSwuLg5dGgAdqx6XHZ+bm2urq6tDl7EnqspS7wBMvZmZmTz11FM5ePDglW3r6+u57rrrcvHixQErg/3Dz519eD7PIb5Wvfd7VZ1urc1ttc8dVABg183OzmZlZeU7tq2srGR2dnagigCG0Vob+9ckE1ABgF23sLCQ+fn5nDp1Kuvr6zl16lTm5+ezsLAwdGkAdMwiSQDArnt6IaQTJ07kzJkzmZ2dzeLiogWSAHhO5qCOmbkAAACMg5876ZU5qAAAAHRPQAUAAKALAioAAABdEFABAADogoAKAABAFwRUAAAAuiCgAgAA0AUBFQAAgC4IqADAnlheXs7Ro0czMzOTo0ePZnl5eeiSAOjcgaELAACmz/LychYWFrK0tJRbb701KysrmZ+fT5IcP3584OoA6JU7qADArltcXMzS0lKOHTuWgwcP5tixY1laWsri4uLQpQHQsWqtDV3DM8zNzbXV1dWhy9gTVZUe33MA2E0zMzN56qmncvDgwSvb1tfXc9111+XixYsDVgb7h5876VVVnW6tzW21zx1UAGDXzc7OZmVl5Tu2raysZHZ2dqCKAJgEAioAsOsWFhYyPz+fU6dOZX19PadOncr8/HwWFhaGLg2AjlkkCQDYdU8vhHTixImcOXMms7OzWVxctEASAM/JHNQxMxcAAIBx8HMnvTIHFQCAkXmGLTAUQ3wBALjCM2yBIbmDCgDAFZ5hCwzJHNQxMxcAAOiZZ9j2p6rGfk0/r7KXzEEFAGAknmHbn9ba2L9gKAIqAABXeIYtMCSLJAEAcIVn2AJDMgd1zMxBBQAA9jNzUAEAAOiegAoAAEAXBFQAAAC6IKACAADQBQEVAACALgioAAAAdGGkgFpVt1XVY1V1tqre/yzH/N2qerCqHq2q39m0/StV9fDGvul8dgwAAADP24HtDqiqmSQfTfKWJOeS3F9V97TWvrTpmO9O8m+S3NZa+2pV/bWr/phjrbUnd69sAAAAps0od1Bfn+Rsa+3LrbVvJflUkndcdcw/TvLZ1tpXk6S19sTulgkAAMC0GyWgviLJ1za9PrexbbO/keRQVf12VZ2uqn+2aV9Lct/G9jueX7kAAABMq22H+CapLba1Lf6cv5nkzUlenOT3q+oLrbX/keSNrbXHN4b9/lZV/WFr7XefcZHL4fWOJPne7/3enbQBAACAKTDKHdRzSV656fXNSR7f4ph7W2tf35hr+rtJfjBJWmuPb/z6RJK7cnnI8DO01j7WWptrrc0dOXJkZ60AAABg4o0SUO9P8uqqelVVvTDJO5Pcc9Uxdyf5O1V1oKpekuSHk5ypquur6qVJUlXXJ3lrkkd2r3wAAACmxbZDfFtr366q9yb5fJKZJJ9orT1aVXdu7D/ZWjtTVfcm+WKSS0k+3lp7pKq+L8ldVfX0tX61tXbvXjUGAACAyVWtXT2ddHhzc3NtdXU6H5laVenxPQcAABiHqjrdWpvbat8oQ3wBAABgzwmoAAAAdEFABabe8vJyjh49mpmZmRw9ejTLy8tDl8SI9B0A7C+jPAcVYGItLy9nYWEhS0tLufXWW7OyspL5+fkkyfHjxweujuei7wBg/7FI0phZJAnG6+jRo/nlX/7lHDt27Mq2U6dO5cSJE3nkEU+96pm+A4Dp9FyLJAmoYyagwnjNzMzkqaeeysGDB69sW19fz3XXXZeLFy8OWBnb0XcAMJ2s4gvsW7Ozs1lZWfmObSsrK5mdnR2oIkal7wBg/xFQgam2sLCQ+fn5nDp1Kuvr6zl16lTm5+ezsLAwdGlsQ98BwP5jkSRgqj29mM6JEydy5syZzM7OZnFx0SI7E0DfAcD+Yw7qmJmDCgAA7GfmoAIAANA9ARUAAIAuCKgAAAB0QUAFAACgCwIqAAAAXRBQAQAA6IKACgAAQBcEVAAAALogoAIAANAFARUAAIAuCKgAAAB0QUAFAACgCwIqAAAAXRBQAQAA6IKACgAAQBcEVAAAALogoMIIlpeXc/To0czMzOTo0aNZXl4euiQAAJg6B4YuAHq3vLychYWFLC0t5dZbb83Kykrm5+eTJMePHx+4OgAAmB7uoMI2FhcXs7S0lGPHjuXgwYM5duxYlpaWsri4OHRpAAAwVaq1NnQNzzA3N9dWV1eHLuM5VdXYr9ljX+0HMzMzeeqpp3Lw4MEr29bX13Pdddfl4sWLA1YGAACTp6pOt9bmttrnDuo1aq2N/YthzM7OZmVl5Tu2raysZHZ2dqCKAABgOgmosI2FhYXMz8/n1KlTWV9fz6lTpzI/P5+FhYWhSwMAgKlikSTYxtMLIZ04cSJnzpzJ7OxsFhcXLZAEAAC7zBxUAAAAxsYcVAAAALonoAIAANAFARUAAIAuCKgAAAB0QUAFAACgCwIqAAAAXRBQAQAA6IKACgAAQBcEVAAAALogoAIAANAFARUAAIAuCKgAAAB0QUAFAACgCwIqAAAAXRBQAQAA6EK11oau4Rmqai3JHw9dxx65KcmTQxfBNdN/k03/TS59N9n03+TSd5NN/022ae6/v95aO7LVji4D6jSrqtXW2tzQdXBt9N9k03+TS99NNv03ufTdZNN/k22/9p8hvgAAAHRBQAUAAKALAur4fWzoAnhe9N9k03+TS99NNv03ufTdZNN/k21f9p85qAAAAHTBHVQAAAC6IKCOSVV9oqqeqKpHhq6FnauqV1bVqao6U1WPVtX7hq6J0VTVdVX1B1X10Ebf/cLQNbEzVTVTVf+9qv7T0LWwM1X1lap6uKoerKrVoethZ6rqu6vq01X1hxv//v2toWtiNFX1mo3P3dNff1FVPz10XYymqn5m42eWR6pquaquG7qmcTLEd0yq6k1J/jLJf2itHR26Hnamqr4nyfe01h6oqpcmOZ3kH7XWvjRwaWyjqirJ9a21v6yqg0lWkryvtfaFgUtjRFX1z5PMJbmhtfZjQ9fD6KrqK0nmWmvT+hy/qVZV/z7Jf22tfbyqXpjkJa21/zNwWexQVc0k+ZMkP9xa++Oh6+G5VdUrcvlnlR9orf1VVf1akv/cWvuVYSsbH3dQx6S19rtJzg9dB9emtfanrbUHNn7/f5OcSfKKYatiFO2yv9x4eXDjy//MTYiqujnJP0zy8aFrgf2kqm5I8qYkS0nSWvuWcDqx3pzkfwmnE+VAkhdX1YEkL0ny+MD1jJWACjtUVbck+aEk/23gUhjRxhDRB5M8keS3Wmv6bnL86yT/Ismlgevg2rQk91XV6aq6Y+hi2JHvS7KW5N9tDLH/eFVdP3RRXJN3JlkeughG01r7kyS/lOSrSf40yZ+31u4btqrxElBhB6rqu5J8JslPt9b+Yuh6GE1r7WJr7bVJbk7y+qoyzH4CVNWPJXmitXZ66Fq4Zm9srb0uye1J3rMx3YXJcCDJ65L829baDyX5epL3D1sSO7UxNPvtSX596FoYTVUdSvKOJK9K8vIk11fVPxm2qvESUGFEG/MXP5Pkk621zw5dDzu3MTztt5PcNmwljOiNSd6+MY/xU0n+XlX9x2FLYidaa49v/PpEkruSvH7YitiBc0nObRpx8ulcDqxMltuTPNBa+7OhC2Fkfz/JH7XW1lpr60k+m+RvD1zTWAmoMIKNhXaWkpxprf2roethdFV1pKq+e+P3L87lb/x/OGhRjKS19vOttZtba7fk8hC1/9Ja21f/izzJqur6jUXlsjE09K1JrGQ/IVpr/zvJ16rqNRub3pzEwoCT53gM7500X03yhqp6ycbPn2/O5bVP9g0BdUyqajnJ7yd5TVWdq6r5oWtiR96Y5J/m8h2cp5ds/wdDF8VIvifJqar6YpL7c3kOqseVwN57WZKVqnooyR8k+c3W2r0D18TOnEjyyY3vn69N8i+HLYedqKqXJHlLLt+BY0JsjFr4dJIHkjycy3ntY4MWNWYeMwMAAEAX3EEFAACgCwIqAAAAXRBQAQAA6IKACgAAQBcEVAAAALogoAIAANAFARUAAIAuCKgAAAB04f8B3BfZGVj0o7IAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1152x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold, cross_val_score\n",
    "from sklearn.feature_selection import SelectKBest, f_classif\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "\n",
    "def load_dataset(filename):\n",
    "    data = pd.read_csv(filename, header=None)\n",
    "    dataset = data.values\n",
    "\n",
    "    x = dataset[:, :-1]\n",
    "    y = dataset[:,-1]\n",
    "\n",
    "    x = x.astype(str)\n",
    "    return x, y\n",
    "\n",
    " \n",
    "def evaluate_model(model):    \n",
    "    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "    scores = cross_val_score(model, x, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
    "    return scores\n",
    "    \n",
    "x, y = load_dataset(\"C:\\\\Users\\\\talk2_000\\\\Desktop\\\\Python\\\\DatasetFiles\\\\pima-indians-diabetes.csv\")\n",
    "num_features = [i+1 for i in range(x.shape[1])]\n",
    "results = list()    \n",
    "\n",
    "for k in num_features:   \n",
    "    model = LogisticRegression(solver='liblinear')\n",
    "    fs = SelectKBest(score_func=f_classif, k=k)\n",
    "    pipeline = Pipeline(steps=[('anova',fs), ('lr', model)])\n",
    "    # evaluate the model\n",
    "    scores = evaluate_model(pipeline)\n",
    "    results.append(scores)\n",
    "    print('>%d %.3f (%.3f)' % (k, np.mean(scores), np.std(scores)))\n",
    "    \n",
    "plt.figure(figsize=(16,8))\n",
    "plt.boxplot(results, labels=num_features, showmeans=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "986da62a",
   "metadata": {},
   "source": [
    "# Numerical input data and a numerical target variable:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d71957ca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train (670, 100) (670,)\n",
      "Test (330, 100) (330,)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=100, n_informative=10, noise=0.1,\n",
    "random_state=1)\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "print('Train', xtrain.shape, ytrain.shape)\n",
    "print('Test', xtest.shape, ytest.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "2029df5f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature 0: 0.009419\n",
      "Feature 1: 1.018881\n",
      "Feature 2: 1.205187\n",
      "Feature 3: 0.000138\n",
      "Feature 4: 0.167511\n",
      "Feature 5: 5.985083\n",
      "Feature 6: 0.062405\n",
      "Feature 7: 1.455257\n",
      "Feature 8: 0.420384\n",
      "Feature 9: 101.392225\n",
      "Feature 10: 0.387091\n",
      "Feature 11: 1.581124\n",
      "Feature 12: 3.014463\n",
      "Feature 13: 0.232705\n",
      "Feature 14: 0.076281\n",
      "Feature 15: 4.299652\n",
      "Feature 16: 1.497530\n",
      "Feature 17: 0.261242\n",
      "Feature 18: 5.960005\n",
      "Feature 19: 0.523219\n",
      "Feature 20: 0.003365\n",
      "Feature 21: 0.024178\n",
      "Feature 22: 0.220958\n",
      "Feature 23: 0.576770\n",
      "Feature 24: 0.627198\n",
      "Feature 25: 0.350687\n",
      "Feature 26: 0.281877\n",
      "Feature 27: 0.584210\n",
      "Feature 28: 52.196337\n",
      "Feature 29: 0.046855\n",
      "Feature 30: 0.147323\n",
      "Feature 31: 0.368485\n",
      "Feature 32: 0.077631\n",
      "Feature 33: 0.698140\n",
      "Feature 34: 45.744046\n",
      "Feature 35: 2.047376\n",
      "Feature 36: 0.786270\n",
      "Feature 37: 0.996190\n",
      "Feature 38: 2.733533\n",
      "Feature 39: 63.957656\n",
      "Feature 40: 231.885540\n",
      "Feature 41: 1.372448\n",
      "Feature 42: 0.581860\n",
      "Feature 43: 1.072930\n",
      "Feature 44: 1.066976\n",
      "Feature 45: 0.344656\n",
      "Feature 46: 13.951551\n",
      "Feature 47: 3.575080\n",
      "Feature 48: 0.007299\n",
      "Feature 49: 0.004651\n",
      "Feature 50: 1.094585\n",
      "Feature 51: 0.241065\n",
      "Feature 52: 0.355137\n",
      "Feature 53: 0.020294\n",
      "Feature 54: 0.154567\n",
      "Feature 55: 2.592512\n",
      "Feature 56: 0.300175\n",
      "Feature 57: 0.357798\n",
      "Feature 58: 3.060090\n",
      "Feature 59: 0.890357\n",
      "Feature 60: 122.132164\n",
      "Feature 61: 2.029982\n",
      "Feature 62: 0.091551\n",
      "Feature 63: 1.081123\n",
      "Feature 64: 0.056041\n",
      "Feature 65: 2.930717\n",
      "Feature 66: 0.054886\n",
      "Feature 67: 1.332787\n",
      "Feature 68: 0.145579\n",
      "Feature 69: 0.986331\n",
      "Feature 70: 0.092661\n",
      "Feature 71: 0.083219\n",
      "Feature 72: 0.198847\n",
      "Feature 73: 2.065792\n",
      "Feature 74: 0.236594\n",
      "Feature 75: 0.512608\n",
      "Feature 76: 1.095650\n",
      "Feature 77: 0.015359\n",
      "Feature 78: 2.193730\n",
      "Feature 79: 1.574530\n",
      "Feature 80: 5.360863\n",
      "Feature 81: 0.041874\n",
      "Feature 82: 5.717705\n",
      "Feature 83: 0.436560\n",
      "Feature 84: 5.594438\n",
      "Feature 85: 0.000065\n",
      "Feature 86: 0.026748\n",
      "Feature 87: 0.408422\n",
      "Feature 88: 2.092557\n",
      "Feature 89: 9.568498\n",
      "Feature 90: 0.642445\n",
      "Feature 91: 0.065794\n",
      "Feature 92: 198.705931\n",
      "Feature 93: 0.073807\n",
      "Feature 94: 1.048605\n",
      "Feature 95: 0.004106\n",
      "Feature 96: 0.042110\n",
      "Feature 97: 0.034228\n",
      "Feature 98: 0.792433\n",
      "Feature 99: 0.015365\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsYAAAHSCAYAAADvxw2lAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAATqklEQVR4nO3dX4yl533Q8e8PbxtoAiLBm8i1DRuQVZogOUErEwhCgQAJparDRWAjtbIglblIIUG1ICkXgQtLuTCBXpBKJgmxRJvFalNiVaE0MkUBCSXZtDWN41qxkpBsbewtBRpxkeL04WJOlKm9213vzNmZnfl8pNU55znvOfOMn/nzndfvOe+stQIAgOPu9x30BAAA4DAQxgAAkDAGAIBKGAMAQCWMAQCgEsYAAFDViYOeQNWNN964Tp06ddDTAADgiPvc5z73G2utkxe771CE8alTpzp37txBTwMAgCNuZv77pe5zKAUAACSMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFDViYOeAMB+uf2+M88be+SeswcwEwCuR/YYAwBAwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIDqCsJ4Zm6dmV+cmcdm5tGZeedm/GUz88mZ+eLm8qW7HvOemXliZh6fmTdt8xMAAID9cCV7jJ+tfnSt9b3V66p3zMyrqndXD6+1bqse3txuc9+Z6tXVm6sPzMwN25g8AADsl8uG8VrrqbXWL22uf716rLq5urN6YLPZA9VbNtfvrM6utb6x1vpy9UR1xz7PGwAA9tULOsZ4Zk5Vr60+Xb1irfVU7cRz9fLNZjdXX9v1sPObMQAAOLSuOIxn5iXVz1TvWmv91u+16UXG1kWe7+6ZOTcz5y5cuHCl0wAAgK24ojCeme9oJ4p/cq31sc3w0zNz0+b+m6pnNuPnq1t3PfyW6snnPuda6/611um11umTJ09e7fwBAGBfnLjcBjMz1Yeqx9Za799110PVXdX7Npcf3zX+UzPz/uq7q9uqz+znpAEAjrvb7zvzvLFH7jl7ADM5Oi4bxtXrqx+qfnVmfmUz9mPtBPGDM/P26qvVW6vWWo/OzIPVF9p5R4t3rLW+ud8TBwCA/XTZMF5r/Zcuftxw1Rsv8Zh7q3v3MC8AALimnPkOAAASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAdQVhPDMfnplnZubzu8b+ycz8+sz8yubf9+267z0z88TMPD4zb9rWxAEAYD9dyR7jj1Rvvsj4P19rvWbz7xNVM/Oq6kz16s1jPjAzN+zXZAEAYFsuG8ZrrU9Vv3mFz3dndXat9Y211perJ6o79jA/AAC4JvZyjPGPzMx/2xxq8dLN2M3V13Ztc34zBgAAh9rVhvFPVH+iek31VPXPNuNzkW3XxZ5gZu6emXMzc+7ChQtXOQ0AANgfVxXGa62n11rfXGv9TvWv+vbhEuerW3dtekv15CWe4/611um11umTJ09ezTQAAGDfXFUYz8xNu27+jepb71jxUHVmZl40M6+sbqs+s7cpAgDA9p243AYz89HqDdWNM3O+em/1hpl5TTuHSXyl+rtVa61HZ+bB6gvVs9U71lrf3MrMAQBgH102jNdab7vI8Id+j+3vre7dy6QAAOBac+Y7AABIGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqOrEQU8AAF6I2+8787yxR+45ewAzAY4ae4wBACBhDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQCWMAQCgEsYAAFAJYwAAqIQxAABUwhgAACphDAAA1RWE8cx8eGaemZnP7xp72cx8cma+uLl86a773jMzT8zM4zPzpm1NHAAA9tOV7DH+SPXm54y9u3p4rXVb9fDmdjPzqupM9erNYz4wMzfs22wBAGBLTlxug7XWp2bm1HOG76zesLn+QPWfqn+0GT+71vpG9eWZeaK6o/qv+zRfXoDb7zvzvLFH7jl7ADMBADj8rvYY41estZ6q2ly+fDN+c/W1Xdud34wBAMChtt8vvpuLjK2Lbjhz98ycm5lzFy5c2OdpAADAC3O1Yfz0zNxUtbl8ZjN+vrp113a3VE9e7AnWWvevtU6vtU6fPHnyKqcBAAD742rD+KHqrs31u6qP7xo/MzMvmplXVrdVn9nbFAEAYPsu++K7mfloOy+0u3Fmzlfvrd5XPTgzb6++Wr21aq316Mw8WH2herZ6x1rrm1uaOwAA7JsreVeKt13irjdeYvt7q3v3MikAALjWnPkOAAASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFR14qAnAPBC3X7fmeeNPXLP2QOYCQBHiT3GAACQMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgEoYAwBAJYwBAKCqE3t58Mx8pfp69c3q2bXW6Zl5WfVvq1PVV6q/udb6X3ubJnD7fWd+1+1H7jl7QDMBgKNpP/YY/8W11mvWWqc3t99dPbzWuq16eHMbAAAOtW0cSnFn9cDm+gPVW7bwMQAAYF/tNYxX9Qsz87mZuXsz9oq11lNVm8uXX+yBM3P3zJybmXMXLlzY4zQAAGBv9nSMcfX6tdaTM/Py6pMz82tX+sC11v3V/VWnT59ee5wHAADsyZ72GK+1ntxcPlP9bHVH9fTM3FS1uXxmr5MEAIBtu+ownpkXz8wf/Nb16q9Wn68equ7abHZX9fG9ThIAALZtL4dSvKL62Zn51vP81Frr52fms9WDM/P26qvVW/c+TQAA2K6rDuO11peq2y8y/j+rN+5lUgAAcK058x0AACSMAQCg2vvbtQHH1HNPUV1OUw3A9c0eYwAASBgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAJUwBgCAShgDAEAljAEAoBLGAABQCWMAAKiEMQAAVMIYAAAqYQwAAFWdOOgJABxVt9935nljj9xz9gBmAsCVsMcYAAASxgAAUAljAACohDEAAFTCGAAAKmEMAACVMAYAgMr7GAMAXFPPfY9z729+eNhjDAAACWMAAKiEMQAAVMIYAAAqYQwAAJV3pTgwz31FanlV6gvhFb0AwH4TxgDAofNCdyBte3uOB4dSAABA9hhzDfnrHAA4zIQxwDXmj0SAw0kYc1l+iQPb5MW0wGEhjIFDSzAdXsfxD+bj+DnDcbO1MJ6ZN1c/Xt1QfXCt9b5tfSzg8BAPHBRfe5d3kP+NjuMfusfxc77ebSWMZ+aG6l9Wf6U6X312Zh5aa31hGx/vOLgWP8wO6gemX2Zsm68xDoKvuysjHjlMtrXH+I7qibXWl6pm5mx1Z3XdhPGlfqD5QQdX53r6njoKcz0Ktv25Hcb/dodxThdzvcyT3+0gdrJdb18X2wrjm6uv7bp9vvozW/pYx94L/SLcr2+Mw/hLa9uf20GN7ydvgv9t18v3wtU4qK+x6+mPiqNsv77Pj8K6HcbfVdfiubg6s9ba/yedeWv1prXWD29u/1B1x1rr7+3a5u7q7s3N76ke3/eJvDA3Vr9xwHNg+6zz8WCdjwfrfDxY5+PhWq7zH1trnbzYHdvaY3y+unXX7VuqJ3dvsNa6v7p/Sx//BZuZc2ut0wc9D7bLOh8P1vl4sM7Hg3U+Hg7LOm/rlNCfrW6bmVfOzHdWZ6qHtvSxAABgz7ayx3it9ezM/Ej1H9p5u7YPr7Ue3cbHAgCA/bC19zFea32i+sS2nn8LDs1hHWyVdT4erPPxYJ2PB+t8PByKdd7Ki+8AAOB6s61jjAEA4LoijNs5ffXMPD4zT8zMuw96PuyPmbl1Zn5xZh6bmUdn5p2b8ZfNzCdn5ouby5ce9FzZm5m5YWZ+eWZ+bnPbGh9BM/OHZ+anZ+bXNt/Xf9ZaHy0z8w82P68/PzMfnZnfb42Phpn58Mw8MzOf3zV2ybWdmfdsuuzxmXnTtZrnsQ/jXaev/mvVq6q3zcyrDnZW7JNnqx9da31v9brqHZu1fXf18FrrturhzW2ub++sHtt12xofTT9e/fxa609Wt7ez5tb6iJiZm6u/X51ea/2pdl68fyZrfFR8pHrzc8Yuurab39VnqldvHvOBTa9t3bEP43advnqt9dvVt05fzXVurfXUWuuXNte/3s4v0ZvbWd8HNps9UL3lQCbIvpiZW6q/Xn1w17A1PmJm5g9Vf6H6UNVa67fXWv87a33UnKj+wMycqL6rnXMgWOMjYK31qeo3nzN8qbW9szq71vrGWuvL1RPt9NrWCeOLn7765gOaC1syM6eq11afrl6x1nqqduK5evkBTo29+xfVP6x+Z9eYNT56/nh1ofrXm8NmPjgzL85aHxlrrV+v7qu+Wj1V/Z+11i9kjY+yS63tgbWZMK65yJi36jhCZuYl1c9U71pr/dZBz4f9MzPfXz2z1vrcQc+FrTtR/enqJ9Zar63+b/6X+pGyOb70zuqV1XdXL56ZHzzYWXFADqzNhPEVnL6a69fMfEc7UfyTa62PbYafnpmbNvffVD1zUPNjz15f/cDMfKWdw6D+0sz8m6zxUXS+Or/W+vTm9k+3E8rW+uj4y9WX11oX1lr/r/pY9eeyxkfZpdb2wNpMGDt99ZE1M9PO8YiPrbXev+uuh6q7Ntfvqj5+refG/lhrvWetdcta61Q737v/ca31g1njI2et9T+qr83M92yG3lh9IWt9lHy1et3MfNfm5/cb23ltiDU+ui61tg9VZ2bmRTPzyuq26jPXYkJO8FHNzPe1c5zit05ffe/Bzoj9MDN/vvrP1a/27eNPf6yd44wfrP5oOz+I37rWeu4LArjOzMwbqnvWWt8/M38ka3zkzMxr2nmR5XdWX6r+djs7eKz1ETEz/7T6W+28q9AvVz9cvSRrfN2bmY9Wb6hurJ6u3lv9uy6xtjPzj6u/087XwrvWWv/+msxTGAMAgEMpAACgEsYAAFAJYwAAqIQxAABUwhgAACphDAAAlTAGAIBKGAMAQFX/H4jZNLgkDLUGAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectKBest, f_regression\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=f_regression, k='all')\n",
    "\n",
    "    fs.fit(xtrain, ytrain)\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=100, n_informative=10, noise=0.1, random_state=1)\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain, ytrain, xtest)\n",
    "\n",
    "for i in range(len(fs.scores_)):\n",
    "    print('Feature %d: %f' % (i, fs.scores_[i]))\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.bar([i for i in range(len(fs.scores_))], fs.scores_, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "7bd995ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature 0: 0.045484\n",
      "Feature 1: 0.000000\n",
      "Feature 2: 0.000000\n",
      "Feature 3: 0.000000\n",
      "Feature 4: 0.024816\n",
      "Feature 5: 0.000000\n",
      "Feature 6: 0.022659\n",
      "Feature 7: 0.000000\n",
      "Feature 8: 0.000000\n",
      "Feature 9: 0.074320\n",
      "Feature 10: 0.000000\n",
      "Feature 11: 0.000000\n",
      "Feature 12: 0.000000\n",
      "Feature 13: 0.000000\n",
      "Feature 14: 0.020390\n",
      "Feature 15: 0.004307\n",
      "Feature 16: 0.000000\n",
      "Feature 17: 0.000000\n",
      "Feature 18: 0.016566\n",
      "Feature 19: 0.003688\n",
      "Feature 20: 0.007579\n",
      "Feature 21: 0.018640\n",
      "Feature 22: 0.025206\n",
      "Feature 23: 0.017967\n",
      "Feature 24: 0.069173\n",
      "Feature 25: 0.000000\n",
      "Feature 26: 0.022232\n",
      "Feature 27: 0.000000\n",
      "Feature 28: 0.007849\n",
      "Feature 29: 0.012849\n",
      "Feature 30: 0.017402\n",
      "Feature 31: 0.008083\n",
      "Feature 32: 0.047321\n",
      "Feature 33: 0.002829\n",
      "Feature 34: 0.028968\n",
      "Feature 35: 0.000000\n",
      "Feature 36: 0.071652\n",
      "Feature 37: 0.027969\n",
      "Feature 38: 0.000000\n",
      "Feature 39: 0.064796\n",
      "Feature 40: 0.137695\n",
      "Feature 41: 0.008732\n",
      "Feature 42: 0.003983\n",
      "Feature 43: 0.000000\n",
      "Feature 44: 0.009387\n",
      "Feature 45: 0.000000\n",
      "Feature 46: 0.038385\n",
      "Feature 47: 0.000000\n",
      "Feature 48: 0.000000\n",
      "Feature 49: 0.000000\n",
      "Feature 50: 0.000000\n",
      "Feature 51: 0.000000\n",
      "Feature 52: 0.000000\n",
      "Feature 53: 0.008130\n",
      "Feature 54: 0.041779\n",
      "Feature 55: 0.000000\n",
      "Feature 56: 0.000000\n",
      "Feature 57: 0.000000\n",
      "Feature 58: 0.031228\n",
      "Feature 59: 0.002689\n",
      "Feature 60: 0.146192\n",
      "Feature 61: 0.000000\n",
      "Feature 62: 0.000000\n",
      "Feature 63: 0.000000\n",
      "Feature 64: 0.018194\n",
      "Feature 65: 0.021368\n",
      "Feature 66: 0.046071\n",
      "Feature 67: 0.034707\n",
      "Feature 68: 0.033530\n",
      "Feature 69: 0.002262\n",
      "Feature 70: 0.018332\n",
      "Feature 71: 0.000000\n",
      "Feature 72: 0.000000\n",
      "Feature 73: 0.074876\n",
      "Feature 74: 0.000000\n",
      "Feature 75: 0.004429\n",
      "Feature 76: 0.002617\n",
      "Feature 77: 0.031354\n",
      "Feature 78: 0.000000\n",
      "Feature 79: 0.000000\n",
      "Feature 80: 0.000000\n",
      "Feature 81: 0.033931\n",
      "Feature 82: 0.010400\n",
      "Feature 83: 0.019373\n",
      "Feature 84: 0.000000\n",
      "Feature 85: 0.033191\n",
      "Feature 86: 0.000000\n",
      "Feature 87: 0.028745\n",
      "Feature 88: 0.000000\n",
      "Feature 89: 0.000000\n",
      "Feature 90: 0.000000\n",
      "Feature 91: 0.017698\n",
      "Feature 92: 0.129797\n",
      "Feature 93: 0.000000\n",
      "Feature 94: 0.002171\n",
      "Feature 95: 0.029995\n",
      "Feature 96: 0.000000\n",
      "Feature 97: 0.014428\n",
      "Feature 98: 0.000000\n",
      "Feature 99: 0.000000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHSCAYAAAAezFYoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAY3klEQVR4nO3dcaxed33f8c93NlkLDKUbdyO1s9lIFtSrlhFZWVqmqoJOjQPC/aeaMwFdpsqKlBRSEaHA/kD7Y9r+iBAgZbGikK4RCGuiqLOY17SCog0JMjsJAYIbzQqMmITmVqiAikSw8t0fz4P65Jdr7rF9r+/1va+XZOU+5/zOc39Xx7l++/h3z6nuDgAA8Lf+zkZPAAAANhuRDAAAA5EMAAADkQwAAAORDAAAA5EMAACDnRs9gZW89rWv7T179mz0NAAA2MIeffTRv+rupZX2bcpI3rNnT06dOrXR0wAAYAurqv93vn2WWwAAwEAkAwDAQCQDAMBAJAMAwEAkAwDAQCQDAMBAJAMAwEAkAwDAQCQDAMBAJAMAwEAkAwDAQCQDAMBAJAMAwEAkAwDAQCQDAMBAJAMAwEAkAwDAQCQDAMBAJAMAwGDnRk8AAC7Wdfccftm2J+46tgEzAbYaV5IBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgMCmSq+qmqnqqqs5U1d0r7H9jVX2pqn5cVXetsH9HVT1eVZ9di0kDAMB6WjWSq2pHknuTHEyyP8ktVbV/GPa9JO9Jcs953ua9SU5fwjwBAOCymXIl+YYkZ7r76e5+IcmxJIcWB3T38919MslPxoOraneStyV5YA3mCwAA625KJO9K8szC67PzbVN9JMn7k7x4AccAAMCGmRLJtcK2nvLmVfX2JM9396MTxh6pqlNVdWp5eXnK2wMAwLqYEslnk1y78Hp3kmcnvv+bk7yjqr6V2TKNt1TVJ1Ya2N33d/eB7j6wtLQ08e0BAGDtTYnkk0n2VdXeqroqyeEkx6e8eXd/oLt3d/ee+XGf7+53XvRsAQDgMti52oDuPldVdyR5OMmOJA9295NVddt8/9Gqel2SU0lek+TFqrozyf7u/sH6TR0AANbHqpGcJN19IsmJYdvRhY+/m9kyjJ/1Hl9I8oULniEAAFxmnrgHAAADkQwAAINJyy0ArjTX3XP4ZdueuOvYBswEgCuRK8kAADAQyQAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADAQyQAAMBDJAAAw2LnREwAA4NJcd8/hl2174q5jGzCTrcOVZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYTIrkqrqpqp6qqjNVdfcK+99YVV+qqh9X1V0L26+tqj+vqtNV9WRVvXctJw8AAOth52oDqmpHknuT/KskZ5OcrKrj3f2NhWHfS/KeJL81HH4uyfu6+7Gq+ntJHq2qPxuOBQCATWXKleQbkpzp7qe7+4Ukx5IcWhzQ3c9398kkPxm2P9fdj80//mGS00l2rcnMAQBgnUyJ5F1Jnll4fTYXEbpVtSfJm5I8cqHHAgDA5TQlkmuFbX0hn6SqXp3kj5Lc2d0/OM+YI1V1qqpOLS8vX8jbAwDAmpoSyWeTXLvweneSZ6d+gqp6RWaB/Mnu/sz5xnX3/d19oLsPLC0tTX17AABYc1Mi+WSSfVW1t6quSnI4yfEpb15VleTjSU5394cvfpoAAHD5rHp3i+4+V1V3JHk4yY4kD3b3k1V123z/0ap6XZJTSV6T5MWqujPJ/iT/LMm7knytqr4yf8sPdveJNf9KAABgjawayUkyj9oTw7ajCx9/N7NlGKMvZuU1zQAAsGl54h4AAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMJkVyVd1UVU9V1ZmqunuF/W+sqi9V1Y+r6q4LORYAADabVSO5qnYkuTfJwST7k9xSVfuHYd9L8p4k91zEsQAAsKlMuZJ8Q5Iz3f10d7+Q5FiSQ4sDuvv57j6Z5CcXeiwAAGw2UyJ5V5JnFl6fnW+bYvKxVXWkqk5V1anl5eWJbw8AAGtvSiTXCtt64vtPPra77+/uA919YGlpaeLbAwDA2psSyWeTXLvweneSZye+/6UcCwAAG2JKJJ9Msq+q9lbVVUkOJzk+8f0v5VgAANgQO1cb0N3nquqOJA8n2ZHkwe5+sqpum+8/WlWvS3IqyWuSvFhVdybZ390/WOnYdfpaAABgTawayUnS3SeSnBi2HV34+LuZLaWYdCwAAGxmnrgHAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAINJkVxVN1XVU1V1pqruXmF/VdXH5vu/WlXXL+z7/ap6sqq+XlWfqqqfW8svAAAA1tqqkVxVO5Lcm+Rgkv1Jbqmq/cOwg0n2zX8dSXLf/NhdSd6T5EB3/3KSHUkOr9nsAQBgHUy5knxDkjPd/XR3v5DkWJJDw5hDSR7qmS8nubqqrpnv25nk56tqZ5JXJnl2jeYOAADrYkok70ryzMLrs/Ntq47p7u8kuSfJt5M8l+T73f2nFz9dAABYf1MiuVbY1lPGVNUvZHaVeW+SX0zyqqp654qfpOpIVZ2qqlPLy8sTpgUAAOtjSiSfTXLtwuvdefmSifON+Y0k3+zu5e7+SZLPJPnVlT5Jd9/f3Qe6+8DS0tLU+QMAwJqbEsknk+yrqr1VdVVmP3h3fBhzPMm753e5uDGzZRXPZbbM4saqemVVVZK3Jjm9hvMHAIA1t3O1Ad19rqruSPJwZneneLC7n6yq2+b7jyY5keTmJGeS/CjJrfN9j1TVp5M8luRckseT3L8eXwgAAKyVVSM5Sbr7RGYhvLjt6MLHneT28xz7oSQfuoQ5AgDAZeWJewAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADAQyQAAMJj0WGoAYGu57p7DL9v2xF3HNmAmsDmJ5G3CN0MAgOkstwAAgIFIBgCAgUgGAICBSAYAgIFIBgCAgUgGAICBSAYAgIH7JAMbYrx3t/t2A7CZuJIMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA7eAgzU23toscXszALjSuJIMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAIOdGz0BgEtx3T2HX7btibuObcBMANhKXEkGAICBSAYAgIFIBgCAgUgGAICBSAYAgIFIBgCAgUgGAICBSAYAgIFIBgCAwaRIrqqbquqpqjpTVXevsL+q6mPz/V+tqusX9l1dVZ+uqr+oqtNV9Str+QUAAMBaWzWSq2pHknuTHEyyP8ktVbV/GHYwyb75ryNJ7lvY99Ekf9Ldb0xyXZLTazBvAABYN1OuJN+Q5Ex3P93dLyQ5luTQMOZQkod65stJrq6qa6rqNUl+LcnHk6S7X+juv1676QMAwNqbEsm7kjyz8PrsfNuUMa9PspzkD6rq8ap6oKpedQnzBQCAdTclkmuFbT1xzM4k1ye5r7vflORvkrxsTXOSVNWRqjpVVaeWl5cnTAsAANbHlEg+m+Tahde7kzw7cczZJGe7+5H59k9nFs0v0933d/eB7j6wtLQ0Ze4AALAupkTyyST7qmpvVV2V5HCS48OY40nePb/LxY1Jvt/dz3X3d5M8U1VvmI97a5JvrNXkAQBgPexcbUB3n6uqO5I8nGRHkge7+8mqum2+/2iSE0luTnImyY+S3LrwFr+X5JPzwH562AcAAJvOqpGcJN19IrMQXtx2dOHjTnL7eY79SpIDFz9FAAC4vDxxDwAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABjs3egLA5nPdPYdftu2Ju45twEwAYGO4kgwAAAORDAAAA8stAGANjcuVLFWCK5MryQAAMHAleYEfVgIAIHElGQAAXkYkAwDAQCQDAMBAJAMAwEAkAwDAQCQDAMBAJAMAwEAkAwDAQCQDAMDAE/cAmMRTSYHtxJVkAAAYTIrkqrqpqp6qqjNVdfcK+6uqPjbf/9Wqun7Yv6OqHq+qz67VxAEAYL2sGslVtSPJvUkOJtmf5Jaq2j8MO5hk3/zXkST3Dfvfm+T0Jc8WAAAugylXkm9Icqa7n+7uF5IcS3JoGHMoyUM98+UkV1fVNUlSVbuTvC3JA2s4bwAAWDdTInlXkmcWXp+db5s65iNJ3p/kxZ/1SarqSFWdqqpTy8vLE6YFAADrY0ok1wrbesqYqnp7kue7+9HVPkl339/dB7r7wNLS0oRpAQDA+pgSyWeTXLvweneSZyeOeXOSd1TVtzJbpvGWqvrERc8WAAAugymRfDLJvqraW1VXJTmc5Pgw5niSd8/vcnFjku9393Pd/YHu3t3de+bHfb6737mWXwAAAKy1VR8m0t3nquqOJA8n2ZHkwe5+sqpum+8/muREkpuTnEnyoyS3rt+UAQBgfU164l53n8gshBe3HV34uJPcvsp7fCHJFy54hgBbxPjEOk+rA9i8PHEPAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABpOeuAcAXJrxiYuJpy7CZuZKMgAADEQyAAAMLLcAAK5IlrCwnkQyAGwgoQebk0gGAFiBv8Bsb9YkAwDAQCQDAMBAJAMAwEAkAwDAwA/uAbBp+EGprcX55ErmSjIAAAxEMgAADEQyAAAMrEkGACCJdeSLXEkGAICBSAYAgIHlFsAl889zADO+H24driQDAMBAJAMAwMByC2BT8U+VAGwGriQDAMBAJAMAwEAkAwDAQCQDAMBAJAMAwEAkAwDAwC3gYBVuSQYA249IvozEFgDAlcFyCwAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABi4BdwmN942zi3jAADWn0iGTcBfhgBgc7HcAgAABpOuJFfVTUk+mmRHkge6+z8P+2u+/+YkP0ryb7v7saq6NslDSV6X5MUk93f3R9dw/gBw2XmCKmx9q15JrqodSe5NcjDJ/iS3VNX+YdjBJPvmv44kuW++/VyS93X3LyW5McntKxwLAACbypTlFjckOdPdT3f3C0mOJTk0jDmU5KGe+XKSq6vqmu5+rrsfS5Lu/mGS00l2reH8AQBgzU1ZbrEryTMLr88m+RcTxuxK8txPN1TVniRvSvLIxUyUK5N/kgQArkRTriTXCtv6QsZU1auT/FGSO7v7Byt+kqojVXWqqk4tLy9PmBYAAKyPKZF8Nsm1C693J3l26piqekVmgfzJ7v7M+T5Jd9/f3Qe6+8DS0tKUuQMAwLqYstziZJJ9VbU3yXeSHE7yb4Yxx5PcUVXHMluK8f3ufm5+14uPJznd3R9ew3kDAFwQSwC5EKtGcnefq6o7kjyc2S3gHuzuJ6vqtvn+o0lOZHb7tzOZ3QLu1vnhb07yriRfq6qvzLd9sLtPrOlXAQAAa2jSfZLnUXti2HZ04eNOcvsKx30xK69Xhk3HFQYA4Kc8lhoAzmMz/uV5M84JtiKRDBfJH1QAsHWJZNgGBD3A1uD7+eUz5RZwAACwrYhkAAAYWG7BFc8/PQEAa82VZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYuLsFAAAXbaveZcqVZAAAGIhkAAAYiGQAABhYkwxbyFZdFwYAl5sryQAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADBwCzgAYEtxO0zWgivJAAAwEMkAADCw3ALgIvjnXICtzZVkAAAYuJIMbCuuAAMwhSvJAAAwcCUZtjFXVVmJ3xcAIpltaCsEwFb4GgBgM7PcAgAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGHiYCAKzKQ4zYbkQyG2b8huubLQCwWYhktixXPbgQfr8AsMiaZAAAGIhkAAAYWG4BTGZJAhvF773Nzc+YsBWJZADYhPzFADaWSAYA2Gb8JWx11iQDAMBgUiRX1U1V9VRVnamqu1fYX1X1sfn+r1bV9VOPBQCAzWbVSK6qHUnuTXIwyf4kt1TV/mHYwST75r+OJLnvAo4FAIBNZcqV5BuSnOnup7v7hSTHkhwaxhxK8lDPfDnJ1VV1zcRjAQBgU5kSybuSPLPw+ux825QxU44FAIBNpbr7Zw+o+u0kv9ndvzt//a4kN3T37y2M+R9J/lN3f3H++nNJ3p/k9asdu/AeRzJbqpEkb0jy1CV+bZfitUn+agM/P5eH87w9OM/bg/O8PTjP28PlPM//pLuXVtox5RZwZ5Ncu/B6d5JnJ465asKxSZLuvj/J/RPms+6q6lR3H9joebC+nOftwXneHpzn7cF53h42y3mestziZJJ9VbW3qq5KcjjJ8WHM8STvnt/l4sYk3+/u5yYeCwAAm8qqV5K7+1xV3ZHk4SQ7kjzY3U9W1W3z/UeTnEhyc5IzSX6U5Nafdey6fCUAALBGJj1xr7tPZBbCi9uOLnzcSW6feuwVYFMs+2DdOc/bg/O8PTjP24PzvD1sivO86g/uAQDAduOx1AAAMBDJA4/R3pqq6tqq+vOqOl1VT1bVe+fb/35V/VlV/d/5f39ho+fKpamqHVX1eFV9dv7aOd6Cqurqqvp0Vf3F/P/rX3Gut56q+v359+yvV9WnqurnnOcrX1U9WFXPV9XXF7ad97xW1QfmXfZUVf3m5ZqnSF7gMdpb2rkk7+vuX0pyY5Lb5+f27iSf6+59ST43f82V7b1JTi+8do63po8m+ZPufmOS6zI75871FlJVu5K8J8mB7v7lzG4AcDjO81bwX5PcNGxb8bzO/6w+nOSfzo/5L/NeW3ci+aU8RnuL6u7nuvux+cc/zOwP1F2Znd8/nA/7wyS/tSETZE1U1e4kb0vywMJm53iLqarXJPm1JB9Pku5+obv/Os71VrQzyc9X1c4kr8zsWQvO8xWuu/9Xku8Nm893Xg8lOdbdP+7ub2Z2J7UbLsc8RfJLeYz2NlBVe5K8KckjSf7R/J7emf/3H27g1Lh0H8nsaZ8vLmxzjree1ydZTvIH86U1D1TVq+Jcbynd/Z0k9yT5dpLnMnsGw5/Ged6qzndeN6zNRPJL1Qrb3P5jC6mqVyf5oyR3dvcPNno+rJ2qenuS57v70Y2eC+tuZ5Lrk9zX3W9K8jfxT+5bznxN6qEke5P8YpJXVdU7N3ZWbIANazOR/FJTHsHNFaqqXpFZIH+yuz8z3/yXVXXNfP81SZ7fqPlxyd6c5B1V9a3Mlkq9pao+Eed4Kzqb5Gx3PzJ//enMotm53lp+I8k3u3u5u3+S5DNJfjXO81Z1vvO6YW0mkl/KY7S3qKqqzNYvnu7uDy/sOp7kd+Yf/06S/36558ba6O4PdPfu7t6T2f+7n+/ud8Y53nK6+7tJnqmqN8w3vTXJN+JcbzXfTnJjVb1y/j38rZn9PInzvDWd77weT3K4qv5uVe1Nsi/J/7kcE/IwkUFV3ZzZusafPkb7P27sjFgLVfUvk/zvJF/L365X/WBm65L/W5J/nNk35N/u7vGHCbjCVNWvJ7mru99eVf8gzvGWU1X/PLMf0LwqydNJbs3swo9zvYVU1X9I8q8zu0PR40l+N8mr4zxf0arqU0l+Pclrk/xlkg8l+eOc57xW1b9P8u8y+31wZ3f/z8syT5EMAAAvZbkFAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAz+PxabTe57uTYgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectKBest, f_regression, mutual_info_regression\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=mutual_info_regression, k='all')\n",
    "\n",
    "    fs.fit(xtrain, ytrain)\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=100, n_informative=10, noise=0.1, random_state=1)\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain, ytrain, xtest)\n",
    "\n",
    "for i in range(len(fs.scores_)):\n",
    "    print('Feature %d: %f' % (i, fs.scores_[i]))\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.bar([i for i in range(len(fs.scores_))], fs.scores_, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6f0ce76b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE: 0.086\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=100, n_informative=10, noise=0.1, random_state=1)\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "model = LinearRegression()\n",
    "model.fit(xtrain, ytrain)\n",
    "\n",
    "yhat = model.predict(xtest)\n",
    "# evaluate predictions\n",
    "mae = mean_absolute_error(ytest, yhat)\n",
    "print('MAE: %.3f' % mae)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b2f6eb7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE: 2.740\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectKBest, f_regression, mutual_info_regression\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=f_regression, k=10)\n",
    "\n",
    "    fs.fit(xtrain, ytrain)\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=100, n_informative=10, noise=0.1, random_state=1)\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain, ytrain, xtest)\n",
    "\n",
    "model = LinearRegression()\n",
    "model.fit(xtrain_fs, ytrain)\n",
    "\n",
    "yhat = model.predict(xtest_fs)\n",
    "\n",
    "mae = mean_absolute_error(ytest, yhat)\n",
    "print('MAE: %.3f' % mae)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "1dbe0f40",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE: 0.085\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectKBest, f_regression, mutual_info_regression\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=f_regression, k=88)\n",
    "\n",
    "    fs.fit(xtrain, ytrain)\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=100, n_informative=10, noise=0.1, random_state=1)\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain, ytrain, xtest)\n",
    "\n",
    "model = LinearRegression()\n",
    "model.fit(xtrain_fs, ytrain)\n",
    "\n",
    "yhat = model.predict(xtest_fs)\n",
    "\n",
    "mae = mean_absolute_error(ytest, yhat)\n",
    "print('MAE: %.3f' % mae)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ea4c4d7",
   "metadata": {},
   "source": [
    "## Model built using Mutual Information Features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "0e1564ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE: 0.084\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectKBest, f_regression, mutual_info_regression\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectKBest(score_func=mutual_info_regression, k=88)\n",
    "\n",
    "    fs.fit(xtrain, ytrain)\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=100, n_informative=10, noise=0.1, random_state=1)\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain, ytrain, xtest)\n",
    "\n",
    "model = LinearRegression()\n",
    "model.fit(xtrain_fs, ytrain)\n",
    "\n",
    "yhat = model.predict(xtest_fs)\n",
    "\n",
    "mae = mean_absolute_error(ytest, yhat)\n",
    "print('MAE: %.3f' % mae)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3b846b0",
   "metadata": {},
   "source": [
    "## Tune the Number of Selected Features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "dd48bcda",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best MAE: -0.082\n",
      "Best Config: {'sel__k': 81}\n",
      ">-1.100 with: {'sel__k': 80}\n",
      ">-0.082 with: {'sel__k': 81}\n",
      ">-0.082 with: {'sel__k': 82}\n",
      ">-0.082 with: {'sel__k': 83}\n",
      ">-0.082 with: {'sel__k': 84}\n",
      ">-0.082 with: {'sel__k': 85}\n",
      ">-0.082 with: {'sel__k': 86}\n",
      ">-0.082 with: {'sel__k': 87}\n",
      ">-0.082 with: {'sel__k': 88}\n",
      ">-0.083 with: {'sel__k': 89}\n",
      ">-0.083 with: {'sel__k': 90}\n",
      ">-0.083 with: {'sel__k': 91}\n",
      ">-0.083 with: {'sel__k': 92}\n",
      ">-0.083 with: {'sel__k': 93}\n",
      ">-0.083 with: {'sel__k': 94}\n",
      ">-0.083 with: {'sel__k': 95}\n",
      ">-0.083 with: {'sel__k': 96}\n",
      ">-0.083 with: {'sel__k': 97}\n",
      ">-0.083 with: {'sel__k': 98}\n",
      ">-0.083 with: {'sel__k': 99}\n",
      ">-0.083 with: {'sel__k': 100}\n"
     ]
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import RepeatedKFold, train_test_split\n",
    "from sklearn.feature_selection import SelectKBest, f_regression, mutual_info_regression\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=100, n_informative=10, noise=0.1, random_state=1)\n",
    "\n",
    "cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "\n",
    "model = LinearRegression()\n",
    "fs = SelectKBest(score_func=mutual_info_regression)\n",
    "pipeline = Pipeline(steps=[('sel',fs), ('lr', model)])\n",
    "\n",
    "grid = dict()\n",
    "grid['sel__k'] = [i for i in range(x.shape[1]-20, x.shape[1]+1)]\n",
    "\n",
    "\n",
    "search = GridSearchCV(pipeline, grid, scoring='neg_mean_absolute_error', n_jobs=-1, cv=cv)\n",
    "\n",
    "results = search.fit(x, y)\n",
    "\n",
    "print('Best MAE: %.3f' % results.best_score_)\n",
    "print('Best Config: %s' % results.best_params_)\n",
    "\n",
    "means = results.cv_results_['mean_test_score']\n",
    "params = results.cv_results_['params']\n",
    "\n",
    "for mean, param in zip(means, params):\n",
    "    print('>%.3f with: %r' % (mean, param))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d4745ac3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">100 -0.083 (0.006)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7cAAAHSCAYAAAA3wjzDAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAlnElEQVR4nO3df4ztZ30n9vcH34Umq5L6rjEY8K29lYnWtIoVpmwUCWizTmLoChMiskZb1TRRXCJoA9KmawRiSaNK3ixRlbZkLZewclddHAcIeNUUYixt0krLwnViEgy4Nj8CN3ZtE1D4g8Rg/PSP+V5y7vU5M3Pv3DPn+5l5vaSrOXPO9zx+8+XMPPM+z3POqTFGAAAAoLNnbDoAAAAA7JdyCwAAQHvKLQAAAO0ptwAAALSn3AIAANCecgsAAEB7xzYd4EK65JJLxhVXXLHpGAAAAKzBvffe+7UxxnOW3Xaoyu0VV1yRkydPbjoGAAAAa1BVf7rqNtuSAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKC9Y5sOAABsTlWd833GGGtIAgD7o9wCwBG2qqhWlRILQCu2JQMAANDevsptVR2vqrur6sHp68Urjruuqh6oqoeq6uaF63+rqu6b/n25qu5buO1t0/EPVNVP7icnAAAAh9t+V25vTnLPGOOqJPdM35+hqi5K8p4kr0xydZLXV9XVSTLG+AdjjGvGGNck+WCSD033uTrJDUlenOS6JL8xjQMAAABPs99ye32S26fLtyd5zZJjXprkoTHGF8cY305yx3S/76ntd7P4mSTvXxj3jjHGE2OMLyV5aBoHAAAAnma/5fa5Y4xHkmT6eumSY16Q5KsL35+arlv0siSPjjEePIf7AAAAQJI9vFtyVX08yfOW3PT2Pf43ln3GwNlvv/j6/PWq7V7vs31g1U1JbkqSEydO7DESANDVuX580Rze9bljZoBudi23Y4xrV91WVY9W1WVjjEeq6rIkjy057FSSyxe+f2GShxfGOJbktUlestf7nJXvtiS3JcnW1paZAAAOuY4fX7Qs15zzAnS0323JdyW5cbp8Y5KPLDnmU0muqqorq+qZ2X6jqLsWbr82yefHGKfOGveGqnpWVV2Z5Kokn9xnVgAAAA6pXVdud3FLkjur6ueSfCXJ65Kkqp6f5L1jjFeNMZ6sqjcn+ViSi5K8b4xx/8IYN+TMLckZY9xfVXcm+WySJ5O8aYzx3X1mBYC1s/0UADajDtOkurW1NU6ePLnpGABwho7bT2Vev255Aeagqu4dY2wtu22/K7cAAMB5sNMDLizlFgAANsAbjcGFpdwCMFtWNQCAvVJuAZitjh/5AofBuT6xlGz+ySVPhgHKLcAR4o8/YC86PrHUMTNwYSm3AEeI13cBwLx5Ivr8KbcA58nkA8BRYt47GJ6IPn/KLcB5MvkAcJTY+s3cKbcAAMChZLX5aFFuz+IHAAAADgerzUeLcnuWjj8ACjnLeFwAAHCUKLeHgNf9sYzHBQAAR8kzNh0AAAAA9svKLQfuXLfLJrbMAgAAO1NuOXBe1wwAAFxoyi3sQcdCDgAA6zanRSDlFpiFOf1iBABgb+a0CKTcArMwp1+MAAD0492SAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2ttXua2q41V1d1U9OH29eMVx11XVA1X1UFXdvHD9b1XVfdO/L1fVfdP1V1TVXy7cdut+cgIAAHC4Hdvn/W9Ocs8Y45aptN6c5B8vHlBVFyV5T5IfT3Iqyaeq6q4xxmfHGP9g4bhfS/IXC3f9whjjmn3mAwAA4AjY77bk65PcPl2+Pclrlhzz0iQPjTG+OMb4dpI7pvt9T1VVkp9J8v595gEAAOAI2m+5fe4Y45Ekmb5euuSYFyT56sL3p6brFr0syaNjjAcXrruyqv6oqn6/ql62z5wAAAAcYrtuS66qjyd53pKb3r7H/0YtuW6c9f3rc+aq7SNJTowx/ryqXpLkw1X14jHGN5fkuynJTUly4sSJPUYCAADgMNm13I4xrl11W1U9WlWXjTEeqarLkjy25LBTSS5f+P6FSR5eGONYktcmecnCf/OJJE9Ml++tqi8keVGSk0vy3ZbktiTZ2to6uzQDAABwBOx3W/JdSW6cLt+Y5CNLjvlUkquq6sqqemaSG6b7nXZtks+PMU6dvqKqnjO9EVWq6m8nuSrJF/eZFQAAgENqv+X2liQ/XlUPZvvdkG9Jkqp6flX9bpKMMZ5M8uYkH0vyuSR3jjHuXxjjhjz9jaRenuSPq+rTST6Q5I1jjK/vMysAAACHVI1xeHbybm1tjZMnn7Zz+YKoqnQ6V93yJjIfhG55k36Zu+VNZD4I3fImMh+EbnkTmQ9Ct7yJzAehW95kfZmr6t4xxtay2/a7cgsAAAAbp9wCAADQnnILAABAe8otAAAA7Sm3AAAAtKfcAgAA0J5yCwAAQHvKLQAAAO0ptwAAALSn3AIAANCecgsAAEB7yi0AAADtKbcAAAC0p9wCAADQnnILAABAe8otAAAA7Sm3AAAAtKfcAgAA0J5yCwAAQHvKLQAAAO0ptwAAALSn3AIAANCecgsAAEB7yi0AAADtKbcAAAC0p9wCAADQnnILAABAe8otAAAA7Sm3AAAAtKfcAgAA0N6RLbfHjx9PVe35X5I9H3v8+PEN/68DAAA4Wo5tOsCmfOMb38gYYy1jny7DAAAAHIwju3ILAADA4aHcAgAA0J5yCwAAQHvKLQAAAO0ptwAAALSn3DZyLh9flPjoIgAA4Og4sh8F1NG6Pr5onR9ddPz48XzjG9+44FkuvvjifP3rXz/fWAAAwCGj3LJWHQs5AADQj23JALtY10sCvCwAAODCsXILC851G3ViK/X56LZdfV07EBK7EAAALhTlFhZ0LDHdimJiuzoAABeecgvNKYos0/FJDwCA/VBuAQ4hT3oAAEeNN5QCAACgPeUWAACA9pRbAAAA2lNuAQAA2NXx48dTVXv6l2TPxx4/fvyC5POGUgAAAAes4ycbzP0NK5VbAACAAzb3otiRbckAAAC0p9wCMAtzfx0PADBvtiUDMAu2ZwEA+2HlFgDOw7pWmte52mx1HIDDzMotAJyHda00J+tbbbY6DsBhZuUWAACA9pRbAGC2bKUGYK9sSwYAZstWagD2ysotAAAA7Sm3AABHmK3fHAbn8jj2WD68bEsGADjCbP3mMOj4DvZceFZuAQAAaE+5BQC4QGzxXT/bT4FVbEsGALhAbPFdP9tPgVWs3AIAANCecgsAAGvUcbt6x8xgWzIAAKxRx+3qHTODlVsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhvX+W2qo5X1d1V9eD09eIVx11XVQ9U1UNVdfPC9ddU1Seq6r6qOllVL1247W3T8Q9U1U/uJycAAACH235Xbm9Ocs8Y46ok90zfn6GqLkryniSvTHJ1ktdX1dXTzb+a5JfHGNckeef0fabbb0jy4iTXJfmNaRwAAAB4mv2W2+uT3D5dvj3Ja5Yc89IkD40xvjjG+HaSO6b7JclI8uzp8g8keXhh3DvGGE+MMb6U5KFpHAAAAHiaY/u8/3PHGI8kyRjjkaq6dMkxL0jy1YXvTyX5u9PltyT5WFW9O9tF+0cX7vOJs+7zgmUBquqmJDclyYkTJ87vfwUAAACt7bpyW1Ufr6rPLPl3/W73PT3EkuvG9PUXkrx1jHF5krcm+c093OfMK8e4bYyxNcbYes5znrPHSAAAABwmu67cjjGuXXVbVT1aVZdNq7aXJXlsyWGnkly+8P0L89fbj29M8ovT5d9O8t493AcAAADOsN/X3N6V7YKa6etHlhzzqSRXVdWVVfXMbL9R1F3TbQ8necV0+ceSPLgw7g1V9ayqujLJVUk+uc+sAAAAHFL7fc3tLUnurKqfS/KVJK9Lkqp6fpL3jjFeNcZ4sqrenORjSS5K8r4xxv3T/X8+ya9X1bEkf5XptbNjjPur6s4kn03yZJI3jTG+u8+sAAAAHFI1xtKXsra0tbU1Tp48uadjqyrr+t++rrG7jbvOsbuNu86xu427zrG7jbvOsbuNu86xu427zrG7jbvOsY27/rG7jbvOsbuNu86xu427zrG7jbvOsc9l3Kq6d4yxtey2/W5LBgAAgI1TbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaU2z14/FuP5w0ffUO+9pdf23QUAAAAllBu9+DWP741f/joH+bWT9+66SgAAAAsodzu4vFvPZ6PPPSRjIx8+KEPt1i9tdIMAAAcNcrtLm7941vz1HgqSfLUeKrF6q2V5oPhSQQAAJgP5XYHp1dtv/PUd5Ik33nqO7Nfve240pz0LIrdnkToeI47ZgYAYDOU2x0srtqeNvfV244rzUnPotjtSYRu5zjpl1kZBwDYHOV2B59+7NPfW7U97TtPfSf3PXbfZgLtouNKc9K3KHZ6EqHjOe6YuVsZBwA4TJTbHXzg1R/In9z4J0/794FXf2DT0ZbquNKc9C2KnZ5E6HaOk36ZO5bxxGozAHB4KLeHSLeV5qR/UTxtzuWr4znumLlbGT/NajMAcFgot4dIt5XmpF9RTPo9idDxHHfL3LGMJ31XmwEAljm26QAcbd2KYpJZP1mwTMdz3C3zTmX8HT/yjg2l2t2y1eY55wUA2Ilyy0Z1K4oddTzH3TJ3K+PJ6tXmN/7QG3PJ912y4XSHy+Pfejy/9Ae/lHe/4t3OLQCskXILsE/dynjSd7U56VcWF1/XPPdzCwCdKbfQ3Pgnz07e9QPrGZdDq+Nq82mdyuLZr2vusjLe7QkEAEiOcLldVyH43thwQOqXv5kxxoUftyrjXRd8WGai42pz0q8sdn1dc6cnEE5TyAE4suV2XYUgUQoA1qVTWez6uuZuTyCc1rGQA3Bh+SggAFro9pFL3T7S6rSOn9nc8WOtHv/W43nDR9/QIitAF0d25RaWsV0d5qvbm2B1fF1z19XmTiv6p1lpBvaq48suNpVZuYUFtqsfDG+CxfnoVhY7vq652xMISc9C3nXrd8c/sOEw6Phk2KYyK7fAgfMmWJyPjmWxm25PICQ9C3nHleak5x/Y0F3HJ8M2mVm5BTiErI5zPjo+gdCtkHdcaU56/oGdWG2mv45Phm0ys3ILcAhZHeeo6FbIO640Jz3/wE76rTZ3LOMyr0/HJ8M2nVm5bcRKDGyGNxo7GH7HcRR0W2lONv/H6vnquNrcrYwnMq9TxyfDNp1ZuW3ESgxshjcaOxh+x3EUdFtpTjb/x+r56rba3LGMy7xeHZ8M23Rm5RYAgJU2/cfq+ei42tytjCcyr1vHJ8M2nbnWtRqxCVtbW+PkyZN7Oraq1rsSs67Vh0bjJlnbVs7tsf/igg/pcbH+cdc5drdx1zl2t3HXOXa3cdc5drdx1zm2cdc/9ibH/ZVP/Ep+58HfOaOU/41n/I289qrX7lhkNpX58W89nld+6JV54rtPfO+6Z130rHz0pz+6Yxnf5ONibpn3Mm7HzHMad51jn8u4VXXvGGNr2W1Wblkr2wyBw8prsVnGa8fnodtqc8et3zIzR8otAJwHr8U+GN3KYscndbud473Y9NbIc9WtjCcyM0+2Ja/BHJbr5zDuOsfuNu46x+427jrH7jbuOsfuNu46x+427jrH7jbuOsc27vrH7jbuOsfuNu46x+427jrH7jbuOse2LRkAgCPHSwKAVazcrsEcntGYw7jrHLvbuOscu9u46xy727jrHLvbuOscu9u4ax272Zv8Jf3+/+s27jrH7jbuWsdu+LPXLXPHx0W3cdc59oVauVVu12AO/6fPYdx1jt1t3HWO3W3cdY7dbdx1jt1t3HWO3W3cdY7dbdx1jm3c9Y/dbdx1jt1t3HWO3W3cdY7dbdx1jn2hyu0zLmgqAAAA2ADlFgAAgPaUWwAAANpTbgEAAGjPRwEBAACt+Yiog7Gu83yhzrFyCwAAtFa//M31vkPwu9YydDvrOs8X6hwrtwAAAAds7qugHSm3AAAAB2zuq6AdeUMpAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2ttXua2q41V1d1U9OH29eMVx11XVA1X1UFXdvHD9NVX1iaq6r6pOVtVLp+uvqKq/nK6/r6pu3U9OAAAADrf9rtzenOSeMcZVSe6Zvj9DVV2U5D1JXpnk6iSvr6qrp5t/NckvjzGuSfLO6fvTvjDGuGb698Z95gQAAOAQ22+5vT7J7dPl25O8ZskxL03y0Bjji2OMbye5Y7pfkowkz54u/0CSh/eZBwAAgCPo2D7v/9wxxiNJMsZ4pKouXXLMC5J8deH7U0n+7nT5LUk+VlXvznbR/tGF466sqj9K8s0k7xhj/N/7zAoAAMAhtWu5raqPJ3nekpvevsf/Ri25bkxffyHJW8cYH6yqn0nym0muTfJIkhNjjD+vqpck+XBVvXiM8c0l+W5KclOSnDhxYo+RAAAAOEx2LbdjjGtX3VZVj1bVZdOq7WVJHlty2Kkkly98/8L89fbjG5P84nT5t5O8d/pvPpHkienyvVX1hSQvSnJySb7bktyWJFtbW+Ps2wEAADj89vua27uyXVAzff3IkmM+leSqqrqyqp6Z5Ibpfsl2yX3FdPnHkjyYJFX1nOmNqFJVfzvJVUm+uM+sAAAAHFL7fc3tLUnurKqfS/KVJK9Lkqp6fpL3jjFeNcZ4sqrenORjSS5K8r4xxv3T/X8+ya9X1bEkf5Vpe3GSlyf5H6rqySTfTfLGMcbX95kVAACAQ2pf5XaM8edJ/t6S6x9O8qqF7383ye8uOe7/SfKSJdd/MMkH95MNAACAo2O/25IBAABg45RbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaO/YpgPA3FTVWsa9+OKL1zIuAACg3MIZxhjndHxVnfN9AACAC8+2ZAAAANo70iu3tp8Ch9k6fsf5/QYAzNWRLbe2n3KYKDGc7Vx+X/n9BgAcBke23HalxHA2JeZg2OkBm2PuA2AvlNtGupYYf5SwTKfHRdefPVim089e4ucPgL1Tblkrf5SwjMcFbIafPQAOM+UWAOAC6rY6nvTMDHA25RYA4ALpuDreMTPAMsotALNh9QgAOF/KLQCz0HH1yLtow2b42QOWUW4B4Dx0LONwGJzrz5KfPzg6lFsAAFizji+76JiZo025BQCANeq406NjZnjGpgMAAADAfim3AAAAtGdbMgAAAHsy59diK7cAAEB7PiJq/eb+WmzlFgAAaM1HRJF4zS0AAACHgHILAABAe7YlAwAAbMCc35ypI+UWAADggM39zZk6si0ZAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaG9f5baqjlfV3VX14PT14hXHXVdVD1TVQ1V188L1P1RV/7aq/qSq/nVVPXvhtrdNxz9QVT+5n5wAAAAcbvtdub05yT1jjKuS3DN9f4aquijJe5K8MsnVSV5fVVdPN783yc1jjP8kye8k+aXpPlcnuSHJi5Ncl+Q3pnEAAADgafZbbq9Pcvt0+fYkr1lyzEuTPDTG+OIY49tJ7pjulyQ/mOQPpst3J/nphXHvGGM8Mcb4UpKHpnEAAADgafZbbp87xngkSaavly455gVJvrrw/anpuiT5TJJXT5dfl+TyPdwHAAAAznBstwOq6uNJnrfkprfv8b9RS64b09efTfI/V9U7k9yV5Nt7uM/Z+W5KclOSnDhxYo+RAAAAOEx2LbdjjGtX3VZVj1bVZWOMR6rqsiSPLTnsVP56RTZJXpjk4Wnszyf5iWmsFyX5L3a7z5J8tyW5LUm2traWFmAAAAAOt/1uS74ryY3T5RuTfGTJMZ9KclVVXVlVz8z2G0XdlSRVden09RlJ3pHk1oVxb6iqZ1XVlUmuSvLJfWYFAADgkNpvub0lyY9X1YNJfnz6PlX1/Kr63SQZYzyZ5M1JPpbkc0nuHGPcP93/9VX1/yb5fLZXZv/FdJ/7k9yZ5LNJPprkTWOM7+4zKwAAAIdUjXF4dvJubW2NkydPrmXsqkqnc9UtbyLzQeiWN+mXuVveROaD0C1vIvNB6JY3kfkgdMubyHwQuuVN1pe5qu4dY2wtu22/K7cAAACwccotAAAA7Sm3AAAAtKfcAgAA0J5yCwAAQHvKLQAAAO0ptwAAALSn3AIAANCecgsAAEB7yi0AAADtKbcAAAC0p9wCAADQnnILAABAe8otAAAA7R3bdABgParqnK4fY6wzzqHkHAMAzIdyy4Fb9Yf/TrdtuhSca+ZN551LhsOu4zlWyAGAw0q55cB1/GO5Y2ZYxmMZADislNuzdFyhsxIDAAAcdcrtWToWv46ZAQAALiTvlgwAAEB7yi0AAADtKbcAAAC0p9wCAADQnjeUAmah4zuVAwAwH8otMAvKKodFt49nO9cnlpLNZwaAZZRbAGar44r+HDKci255AWAV5RaA2VK8AIC9Um4BgFY6rugDsH7KLQDQirIKwDI+CggAAID2lFsAAADasy0ZAGDNun1EFEBHyi0AwJp1K6sdP/+44xuNedIDLizlFgCAM3QsUTJzWHjS4/wptwAAwKHUcUV/Dhm6Um4BAIBDSVE8WpRbAABgVx1XQVm/OT0ulFsAAGBXyirLzOlx4XNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKA95RYAAID2lFsAAADaU24BAABoT7kFAACgPeUWAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKC9GmNsOsMFU1WPJ/nTNQ1/SZKvrWnsdeiWN5H5IHTLm/TL3C1vIvNB6JY3kfkgdMubyHwQuuVNZD4I3fIm68v8H44xnrPshkNVbtepqk6OMbY2nWOvuuVNZD4I3fIm/TJ3y5vIfBC65U1kPgjd8iYyH4RueROZD0K3vMlmMtuWDAAAQHvKLQAAAO0pt3t326YDnKNueROZD0K3vEm/zN3yJjIfhG55E5kPQre8icwHoVveROaD0C1vsoHMXnMLAABAe1ZuAQAAaE+5XaKq3lpV91fVZ6rq/VX171XV66brnqqq2b1T2YrM/6yqPl9Vf1xVv1NV/8Gmc562Iu+vTFnvq6rfq6rnbzrnomWZF277R1U1quqSTWZctOIcv6uq/mw6x/dV1as2nXPRqnNcVf9tVT0w3farm865aMV5/q2Fc/zlqrpv0zlPW5H3mqr6xJT3ZFW9dNM5F63I/ENV9W+r6k+q6l9X1bM3nXNRVf3ilPf+qnrLdN3xqrq7qh6cvl684ZjfsyLv3Oe9ZZlnO+8lKzPPdu5blnfhttnNe8nKczzbuW/VOZ75vLfsHM923ktWZp7t3Lci76zmvap6X1U9VlWfWbhu5TxXVW+rqoemx/VPri3YGMO/hX9JXpDkS0m+b/r+ziRvSPJ3kvxgkn+TZGvTOfeY+SeSHJuu+6dJ/umms+6S99kLx/x3SW7ddNbdMk+XL0/ysWx/xvIlm866yzl+V5J/tOl855j5P0/y8STPmq6/dNNZ9/K4WDjm15K8c9NZdznHv5fkldN1r0rybzaddQ+ZP5XkFdN1P5vkVzaddSHzf5zkM0m+P8mx6fF7VZJfTXLzdMzNM/qdvCrvnOe9VZlnOe/tknmWc9+qvNNts5v3djnHs5z7dsg753lv5eNi4ZjZzHu7nOdZzn075J3VvJfk5Ul+OMlnFq5bOs8luTrJp5M8K8mVSb6Q5KJ15LJyu9yxJN9XVcey/cB6eIzxuTHGAxvOtZNlmX9vjPHkdPsnkrxwY+meblneby7c/jeTzO0F4U/LPF3/PyX579Mn75wty/wLSW4ZYzyRJGOMxzaYb5mV57mqKsnPJHn/hrItsyzvSHL6GeAfyPweK8sy/2CSP5huvzvJT28o2zJ/J8knxhjfmn4H/36Sn0pyfZLbp2NuT/KazcR7mqV5Zz7vrco853lvVea5zn2rHsfJfOe9nTLP0aq8c573djzHM533VmWe69y3Ku+s5r0xxh8k+fpZV6+a565PcscY44kxxpeSPJRkLSvlyu1Zxhh/luTdSb6S5JEkfzHG+L3NptrZHjP/bJL/66CzLbNT3qr6H6vqq0n+YZJ3bi7lmVZlrqpXJ/mzMcanNxrwLLs8Jt48bYF735y2Re6Q+UVJXlZV/66qfr+q/tNN5ly0h5+9lyV5dIzx4CbynW2HvG9J8s+mn713J3nbxkKeZYfMn0ny6umw12V7JWkuPpPk5VX1t6rq+7O9InB5kueOMR5JkunrpRvMuGhV3jnbS+bZzHuTlZlnOvctzTvXeW+y0+NijnPfqryznfey+8/erOa9yarMb8k8575Veec87522ap57QZKvLhx3arruglNuzzL9wrs+20vmz0/yN6vqv9xsqp3tlrmq3p7kyST/x2YSnmmnvGOMt48xLs921jdvLuWZVmT+r5K8PfP5Q+R7djjH/zzJf5TkmmwXhV/bVMaz7ZD5WJKLk/xIkl9Kcuf0zPDG7eH3xeszo2evd8j7C0neOv3svTXJb24u5Zl2yPyzSd5UVfcm+feTfHtzKc80xvhctrfE3p3ko9neivXkjnfaoG55k90zz23eS3bOPMe5b4e8s5z3kh0zz3Lu2yHvbOe9Pfy+mNW8l+yYeZZz3w55Zzvv7cGyx+9adn4ot093bZIvjTEeH2N8J8mHkvzohjPtZmXmqroxyd9P8g/HtOl9BvZyjv9V5rXNcFnm/zrbf3B/uqq+nO3tb39YVc/bXMzvWXqOxxiPjjG+O8Z4Ksn/ljVtCTlPqx4Xp5J8aGz7ZJKnkszlDUx2+tk7luS1SX5rg/nOtirvjdPlJPntNHhcjDE+P8b4iTHGS7L9h9QXNpryLGOM3xxj/PAY4+XZ3rb1YJJHq+qyJJm+zmar4Yq8s7Yq80znvSR7Os+zmvuW5P1y5jvvJVl+juc89614TMx53tvpZ2+O816SlZlnO/eteBzPet6brJrnTuXMleYXZk3bwJXbp/tKkh+pqu+fniX7e0k+t+FMu1mauaquS/KPk7x6jPGtjSY806q8Vy0c8+okn99IuuWWZf7QGOPSMcYVY4wrsv2D+8NjjP9vk0Enq87xZQvH/FS2t7jMxaqfvQ8n+bEkqaoXJXlmkq9tKuRZdvp9cW2Sz48xTm0s3dOtyvtwkldMx/xY5lVsVj2WL02SqnpGknckuXWDGZ9mId+JbP+x9/4kd2X7j6lMXz+ymXRPtyLvrC3LPON5L8nKzLOd+5bk/d9nPO8lWXmOZzv3rfjZ+3DmO+/t9PtijvNekpWZZzv3rXgcz3rem6ya5+5KckNVPauqrsz2G2R9ch0Bjq1j0M7GGP+uqj6Q5A+zvQXgj5LcVlU/leR/SfKcJP9nVd03xljf21ifg1WZk9yf7Xclu3vazfKJMcYbNxZ0skPef1VVP5jtZyj/NMnGs562Q+ZZ2iHve6vqmmxvBflykv9mUxnPtkPmkeR9tf1W899OcuNcVmN2eVzckJkVhB3y/lGSX5+edf+rJDdtLuWZdsj8xqp603TYh5L8iw1FXOWDVfW3knwnyZvGGN+oqluyvb3w57Jd2l+30YRnWpZ3tvPeZFnm/zUznPcWLMv83rnOfVmSd9OB9mDZOf6Xc537sjzv+zLTeW+y6nExu3lvwbLz/POZ6dyX5Xl/cU7zXlW9P8l/luSSqjqV5J8kWTrPjTHur6o7k3w223P5m8YY311Lrnn9rAAAAMC5sy0ZAACA9pRbAAAA2lNuAQAAaE+5BQAAoD3lFgAAgPaUWwAAANpTbgEAAGhPuQUAAKC9/x9q8SwswiWQ3QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import RepeatedKFold, train_test_split, cross_val_score\n",
    "from sklearn.feature_selection import SelectKBest, f_regression, mutual_info_regression\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=100, n_informative=10, noise=0.1, random_state=1)\n",
    "\n",
    "num_features = [i for i in range(x.shape[1]-19, x.shape[1]+1)]\n",
    "\n",
    "results = list()\n",
    "for k in num_features:\n",
    "    model = LinearRegression()\n",
    "    fs = SelectKBest(score_func=mutual_info_regression, k=k)\n",
    "    pipeline = Pipeline(steps=[('sel',fs), ('lr', model)])\n",
    "\n",
    "    cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "    scores = cross_val_score(pipeline, x, y, scoring='neg_mean_absolute_error', cv=cv, n_jobs=-1)\n",
    "    results.append(scores)\n",
    "\n",
    "print('>%d %.3f (%.3f)' % (k, np.mean(scores), np.std(scores)))\n",
    "\n",
    "plt.figure(figsize=(16,8))\n",
    "plt.boxplot(results, labels=num_features, showmeans=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fec207c5",
   "metadata": {},
   "source": [
    "# How to Use RFE for Feature Selection:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ceb87ac",
   "metadata": {},
   "source": [
    "## RFE for Classification:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "bbb5e415",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.883 (0.032)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import cross_val_score, RepeatedStratifiedKFold\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "x, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, random_state=1)\n",
    "\n",
    "rfe = RFE(estimator=DecisionTreeClassifier(), n_features_to_select=5)\n",
    "model = DecisionTreeClassifier()\n",
    "pipeline = Pipeline(steps=[('s',rfe),('m',model)])\n",
    "\n",
    "cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "n_scores = cross_val_score(pipeline, x, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
    "\n",
    "print('Accuracy: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "6a49e437",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted Class: 1\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import make_classification\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "x, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, random_state=1)\n",
    "\n",
    "rfe = RFE(estimator=DecisionTreeClassifier(), n_features_to_select=5)\n",
    "model = DecisionTreeClassifier()\n",
    "pipeline = Pipeline(steps=[('s',rfe),('m',model)])\n",
    "\n",
    "pipeline.fit(x, y)\n",
    "\n",
    "data = [[2.56999479, -0.13019997, 3.16075093, -4.35936352, -1.61271951, -1.39352057, -2.48924933, -1.93094078, 3.26130366, 2.05692145]]\n",
    "yhat = pipeline.predict(data)\n",
    "print('Predicted Class: %d' % (yhat))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e2bc8ad",
   "metadata": {},
   "source": [
    "## RFE for Regression:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "cc7794fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAE: -27.241 (2.667)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import RepeatedKFold\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=10, n_informative=5, random_state=1)\n",
    "\n",
    "rfe = RFE(estimator=DecisionTreeRegressor(), n_features_to_select=5)\n",
    "model = DecisionTreeRegressor()\n",
    "pipeline = Pipeline(steps=[('s',rfe),('m',model)])\n",
    "\n",
    "cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "n_scores = cross_val_score(pipeline, x, y, scoring='neg_mean_absolute_error', cv=cv, n_jobs=-1)\n",
    "print('MAE: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "1b0383e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted: -84.288\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import RepeatedKFold\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=10, n_informative=5, random_state=1)\n",
    "\n",
    "rfe = RFE(estimator=DecisionTreeRegressor(), n_features_to_select=5)\n",
    "model = DecisionTreeRegressor()\n",
    "pipeline = Pipeline(steps=[('s',rfe),('m',model)])\n",
    "\n",
    "pipeline.fit(x, y)\n",
    "data = [[-2.02220122, 0.31563495, 0.82797464, -0.30620401, 0.16003707, -1.44411381, 0.87616892, -0.50446586, 0.23009474, 0.76201118]]\n",
    "yhat = pipeline.predict(data)\n",
    "\n",
    "print('Predicted: %.3f' % (yhat))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "afaf8c3c",
   "metadata": {},
   "source": [
    "## RFE Hyperparameters:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11d912c5",
   "metadata": {},
   "source": [
    "### Explore number of features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "f0363814",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">2 0.720 (0.046)\n",
      ">3 0.814 (0.029)\n",
      ">4 0.875 (0.030)\n",
      ">5 0.883 (0.025)\n",
      ">6 0.886 (0.027)\n",
      ">7 0.887 (0.027)\n",
      ">8 0.885 (0.024)\n",
      ">9 0.882 (0.027)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHSCAYAAAAezFYoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAgf0lEQVR4nO3dfYyl51kf4N/N7BonTpzsYhPAHzhUaVhrVT46MrRxA9s0wW4hKRShuGpRokWuq8QKVKIEFilBaCUqaFUk0q6sbAq0ZC1w4thUyElEF8KiAp41u/6M260TkmUBj9ktKRjjsffpH3PWDI/H3jOzZ/Y9Z+a6pNH4vF9zn8dnz/zmOff7vtVaCwAA8Ne+bOgCAABg2gjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAACdbUMXsJorrriiXXfddUOXAQDAJnb06NGnWmtXrrZuKkPyddddl4WFhaHLAABgE6uqP3ipddotAACgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdMYKyVV1U1U9XlUnqur9q6zfUVV3V9WDVfV7VbV7xbrPV9VDVXWsqhYmWTwAAGyEbefboKrmknwoyVuTnExyf1Xd21p7dMVmP5bkWGvtu6vq60fbv2XF+j2ttacmWDcAAGyYcWaSb0hyorX2RGvt2SR3JnlHt831SX49SVprn01yXVW9bqKVAgDARTJOSL4qyRdXPD45WrbS8STfkyRVdUOSr01y9WhdS/KpqjpaVbdeWLkAALDxzttukaRWWda6xz+V5Ger6liSh5L8fpLnRuve1Fo7VVVfmeTTVfXZ1tpnXvRDlgP0rUly7bXXjlk+ANOgarVfFReutf7XDcDFMU5IPpnkmhWPr05yauUGrbUvJXl3ktTyO+XnRl9prZ0afX+yqu7OcvvGi0Jya+2OJHckyfz8vHdFgBkybpitKsEXmAnjtFvcn+QNVfX6qrokyTuT3Ltyg6p67WhdkvxAks+01r5UVZdV1atH21yW5G1JHp5c+QAAMHnnnUlurT1XVe9N8skkc0k+0lp7pKpuG60/kGRXkl+squeTPJpk72j31yW5e/Qx3LYkH22t3Tf5pwEAAJNT0/ix1/z8fFtYcEllgM1GuwUwTarqaGttfrV17rgHAAAdIRkAADpCMgAAdIRkAADoCMkAwJZ26NCh7N69O3Nzc9m9e3cOHTo0dElMgXFuJgIAsCkdOnQo+/bty8GDB3PjjTfmyJEj2bt3+Uq2t9xyy8DVMSQzyQDAlrV///4cPHgwe/bsyfbt27Nnz54cPHgw+/fvH7o0BuY6yQBcNK6TfH6jG3BNnHFf3dzcXJ555pls3779hWVLS0u59NJL8/zzzw9Y2XTabK9P10kGgBnRWhv7ay3bs7pdu3blyJEjf2PZkSNHsmvXroEqmm4b8dqc1tenkAwAbFn79u3L3r17c/jw4SwtLeXw4cPZu3dv9u3bN3RpDMyJewDAlnXu5Lzbb789jz32WHbt2pX9+/c7aQ89yQBcPHqSJ8t4Mq1m5bWpJxkAANZASAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQcTMRYEuqqg057ixcFxS2Ev/WWS8hGdiS1vILblYuig+8mH/rrJd2CwAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoLNt6AKA8VTVhhy3tbYhxwWAWSYkw4xYS5itKuEXAC6AdgsAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQcVtqAFa1c+fOnDlzZuLHraqJHm/Hjh05ffr0RI8JICQDsKozZ86ktTZ0Gec16dANkGi3AACAFzGTDAAXgfYVmC1CMgBcBNpXYLZotwAAgI6QDAAAHSEZAAA6QjIAAHTGCslVdVNVPV5VJ6rq/aus31FVd1fVg1X1e1W1e9x9AQBg2pw3JFfVXJIPJbk5yfVJbqmq67vNfizJsdba30ny/Ul+dg37AgDAVBlnJvmGJCdaa0+01p5NcmeSd3TbXJ/k15OktfbZJNdV1evG3BcAAKbKOCH5qiRfXPH45GjZSseTfE+SVNUNSb42ydVj7pvRfrdW1UJVLSwuLo5XPWwCO3fuTFVN9CvJxI+5c+fOgUdqPLMwnrMylsDWMAvvm0O8d45zM5HVrireXw39p5L8bFUdS/JQkt9P8tyY+y4vbO2OJHckyfz8/PRfbR0mxA0GJmsWxnNWxhLYGmbhfTO5+O+d44Tkk0muWfH46iSnVm7QWvtSkncnSS0/g8+Nvl55vn0BAGDajNNucX+SN1TV66vqkiTvTHLvyg2q6rWjdUnyA0k+MwrO590XAACmzXlnkltrz1XVe5N8Mslcko+01h6pqttG6w8k2ZXkF6vq+SSPJtn7cvtuzFMBALaKnTt35syZMxM/7qQ/0t+xY0dOnz490WNycdQ09qDMz8+3hYWFocuAi6KqZqYXTJ2TMQs1JuqcNHVOljonZxZqTDamzqo62lqbX22dO+4BAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQGee21ADAlFl8ejE//Jkfzs9828/kildcMXQ5F137wOXJB18zdBnn1T5w+dAlsE5CMgAXxVYPdZN24MEDeeBPHsiB4wfy49/640OXc9HVT3xpdm6A8cGhq2A9tFsAcFGsDHVcmMWnF3PPiXvS0vKJE5/IU3/51NAlwaYjJAOw4YS6yTrw4IGcbWeTJGfbWX94TMDi04t5133v8trkBUIyABtOqJucc39wLJ1dSpIsnV3yh8cE+KSDnpAMwIYS6iZr5R8c5/jD48L4pIPVOHEP4GVs5ZPNJnX1gANfsSNnX/Wq5MvqhWVnl57JgQ/P58f/9MwFH39Wrh4wqfE8/jVflaUvv+RvLFs6u5RjD/7X5L6fvuDjz8p4TtJqn3RsxZMhJ2WzvG/WNJ4ZOj8/3xYWFoYuAy6KqpqdM7S3YJ0/+Ts/mV95/FfyfW/8von90txqY/m9935vHj/z+IuWv3HHG3PX2++64ONvtfHcaFutzsWnF3Pzx2/OXz3/Vy8s+/K5L899/+y+iQS8WRjPWXjfTDZmLKvqaGttfrV1ZpIBXkL/Eext33DbTM+KDGUSQRg2ysu1r5hNXrvN9L6pJxngJTjZDDa/408ef6Ff/pyls0s59uSxYQqacZvpfVO7BQxsFj6KS7ZenRv5EexWG8uNps7JUudkzUKds/C+mVz8dgszyQCrcAUBgLXZbO+bZpJhaBM42/2i+eCfDV3B+U1oPL/3a74qj3dXEEiSN/7Vs7nr1B9f+A+YgbGchRmwRJ2Tps7Jmok6Z+V9M5n4e+fLzSQLyTCwmXgDjTonaRZqTNQ5aeqcLHVOzizUmGi3AACAwQnJsMksPr2Yd933LneMAoALICTDJnPgwQN54E8emNkTJQBgGgjJsIn0F3E3mwwA6yMkwyaymS7iDgBDEpJhkzg3i3zuzlFLZ5fMJgPAOgnJsElstou4A8CQtg1dADAZx588/sIs8jlLZ5dy7MljwxQEsMGqaugSzmvHjh1Dl8A6CcmwSdz19ruGLgHgotmIm1/Myk01uDi0WwAAQEdIBgCAjnYLNtRG9Yv5OIyXM+19inoUt65pf20mXp9bldfmiwnJbKhxw6w+MCZFnyLTymuTaeW1uTrtFgAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHRcJxmAl+QGA8BWJSQDsCo3GAC2Mu0WAADQMZMMU8BH2gAwXYRkGJiPtAFg+mi3AACAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoDNWSK6qm6rq8ao6UVXvX2X9a6rqV6vqeFU9UlXvXrHu81X1UFUdq6qFSRYPAAAb4bw3E6mquSQfSvLWJCeT3F9V97bWHl2x2XuSPNpa+66qujLJ41X1S621Z0fr97TWnpp08QAAsBHGmUm+IcmJ1toTo9B7Z5J3dNu0JK+u5XvrvirJ6STPTbRSAAC4SMYJyVcl+eKKxydHy1b6uSS7kpxK8lCS97XWzo7WtSSfqqqjVXXrS/2Qqrq1qhaqamFxcXHsJwAA8FKqauyvtWzP5jdOSF7tldC6x9+R5FiSr0nyjUl+rqouH617U2vtm5PcnOQ9VfXm1X5Ia+2O1tp8a23+yiuvHKd2AICX1VrbkC82v3FC8skk16x4fHWWZ4xXeneSj7dlJ5J8LsnXJ0lr7dTo+5NJ7s5y+wYAAEytcULy/UneUFWvr6pLkrwzyb3dNl9I8pYkqarXJXljkieq6rKqevVo+WVJ3pbk4UkVDwAAG+G8V7dorT1XVe9N8skkc0k+0lp7pKpuG60/kOQnk/x8VT2U5faMH2mtPVVVX5fk7lHvzrYkH22t3bdBzwUAACbivCE5SVprv5bk17plB1b896kszxL3+z2R5BsusEYAALio3HEPAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAzljXSQbYbEY3OZr49q219ZQz89YynmvZdquOJzA8IRnYkoSvyTKewGaj3QIAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB03JaaNdu5c2fOnDkz8eNW1USPt2PHjpw+fXqixxzSWsdn3O3dThimi3/rTLO1vD7Xsu00vj6FZNbszJkzU/li7k06dA9tFsYcuHD+rTPNttLrU7sFAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZANjSDh06lN27d2dubi67d+/OoUOHhi6JKeBmIgDAlnXo0KHs27cvBw8ezI033pgjR45k7969SZJbbrll4OoYkplkAGDL2r9/fw4ePJg9e/Zk+/bt2bNnTw4ePJj9+/cPXRoDq2m8veD8/HxbWFgYugxeQlXNxG0pZ6VOAIYzNzeXZ555Jtu3b39h2dLSUi699NI8//zzA1bGxVBVR1tr86utM5MMAGxZu3btypEjR/7GsiNHjmTXrl0DVcS0EJIBgC1r37592bt3bw4fPpylpaUcPnw4e/fuzb59+4YujYE5cQ8A2LLOnZx3++2357HHHsuuXbuyf/9+J+2hJ5m1m5Ve31mpEwAYhp5kptri04t5133vylN/+dTQpQAAJNFuwTq0D1yefPA1Ezvega/YkQde/aoc+PB8fvxPz0zsuO0Dl0/sWADA1iIks2b1E1+aWBvD4tOLuefjN6c9/1f5xI4rctsPLOSKV1wxkWNXVdoHJ3IoAGCL0W7BoA48eCBn29kkydl2NgeOHxi4IgAAIZkBLT69mHtO3JOls0tJkqWzS/nEiU/oTQYABickM5iVs8jnmE0GAKaBkMxgjj95/IVZ5HOWzi7l2JPHhikIAGDEiXsM5q633zV0CQAAqxKSO1W1Icd1UwsAgNkhJHfGDbPu5gYAsHnpSQYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQGSskV9VNVfV4VZ2oqvevsv41VfWrVXW8qh6pqnePuy8AAEyb84bkqppL8qEkNye5PsktVXV9t9l7kjzaWvuGJN+e5N9X1SVj7gsAAFNlnJnkG5KcaK090Vp7NsmdSd7RbdOSvLqWb1f3qiSnkzw35r4AADBVxgnJVyX54orHJ0fLVvq5JLuSnEryUJL3tdbOjrkvAABMlXFuS12rLOvvx/wdSY4l+YdJ/laST1fVb4257/IPqbo1ya1Jcu21145RFkNa/tBguu3YsWPoEgCAGTVOSD6Z5JoVj6/O8ozxSu9O8lOttZbkRFV9LsnXj7lvkqS1dkeSO5Jkfn5+1SDNdFj+3zxZVbUhxwUAWI9x2i3uT/KGqnp9VV2S5J1J7u22+UKStyRJVb0uyRuTPDHmvgAAMFXOO5PcWnuuqt6b5JNJ5pJ8pLX2SFXdNlp/IMlPJvn5qnooyy0WP9JaeypJVtt3Y54KAABMRk3jR9zz8/NtYWFh6DJelvaAyTKeAMDFVlVHW2vzq61zxz0AAOgIyQAA0BGSAQCgIyQDAEBHSIZN5NChQ9m9e3fm5uaye/fuHDp0aOiSAGAmjXMzEWAGHDp0KPv27cvBgwdz44035siRI9m7d2+S5JZbbhm4OgCYLWaSYZPYv39/Dh48mD179mT79u3Zs2dPDh48mP379w9dGgDMHNdJXifX9Z0s43nh5ubm8swzz2T79u0vLFtaWsqll16a559/fsDKAGA6uU4ybAG7du3KkSNH/sayI0eOZNeuXQNVBACzS0iGTWLfvn3Zu3dvDh8+nKWlpRw+fDh79+7Nvn37hi4NAGaOE/dgkzh3ct7tt9+exx57LLt27cr+/fudtAcA66AneZ300E6W8QQALjY9yQAAsAZCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAJ1tQxdwsezcuTNnzpyZ6DGraqLH27FjR06fPj3RYw5tLWO0lm1ba+spBwBgLFsmJJ85c2bqg9WkQ/c0mPYxBwBYjXYLAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOmOF5Kq6qaoer6oTVfX+Vdb/cFUdG309XFXPV9XO0brPV9VDo3ULk34CAAAwaee9mUhVzSX5UJK3JjmZ5P6qure19ui5bVprP53kp0fbf1eSH2qtrbx13J7W2lMTrRwAADbIODPJNyQ50Vp7orX2bJI7k7zjZba/JcmhSRQHAABDGCckX5Xkiysenxwte5GqemWSm5J8bMXiluRTVXW0qm5db6EAAHCxnLfdIkmtsqy9xLbfleS3u1aLN7XWTlXVVyb5dFV9trX2mRf9kOUAfWuSXHvttWOUBQAAG2OcmeSTSa5Z8fjqJKdeYtt3pmu1aK2dGn1/MsndWW7feJHW2h2ttfnW2vyVV145RlkAALAxxgnJ9yd5Q1W9vqouyXIQvrffqKpek+TbktyzYtllVfXqc/+d5G1JHp5E4QAAsFHO227RWnuuqt6b5JNJ5pJ8pLX2SFXdNlp/YLTpdyf5VGvtL1bs/rokd1fVuZ/10dbafZN8AgAAMGnV2ku1Fw9nfn6+LSxM9pLKVZVpfK4rzUKNAACbRVUdba3Nr7bOHfcAAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIXkdFp9ezLvue1ee+sunhi4FAIANICSvw4EHD+SBP3kgB44fGLoUAAA2gJC8RotPL+aeE/ekpeUTJz5hNhkAYBMSktfowIMHcradTZKcbWfNJgMAbEJC8hqcm0VeOruUJFk6u2Q2GQBgExKS12DlLPI5ZpMBADYfIXkNjj95/IVZ5HOWzi7l2JPHhikIAIANsW3oAmbJXW+/a+gSAAC4CMwkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgs2VuJtI+cHnywdcMXcbLah+4fOgSAADIFgrJ9RNfSmtt6DJeVlWlfXDoKgAA0G4BAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAZ6yQXFU3VdXjVXWiqt6/yvofrqpjo6+Hq+r5qto5zr4AADBtzhuSq2ouyYeS3Jzk+iS3VNX1K7dprf10a+0bW2vfmORHk/xma+30OPsCAMC0GWcm+YYkJ1prT7TWnk1yZ5J3vMz2tyQ5tM59AQBgcOOE5KuSfHHF45OjZS9SVa9MclOSj611XwAAmBbjhORaZVl7iW2/K8lvt9ZOr3Xfqrq1qhaqamFxcXGMsgAAYGOME5JPJrlmxeOrk5x6iW3fmb9utVjTvq21O1pr8621+SuvvHKMsgAAYGOME5LvT/KGqnp9VV2S5SB8b79RVb0mybcluWet+wIAwDTZdr4NWmvPVdV7k3wyyVySj7TWHqmq20brD4w2/e4kn2qt/cX59p30kwAAgEmq1l6qvXg48/PzbWFhYaLHrKpM43NdaRZqBADYLKrqaGttfrV17rgHAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHS2DV3AxVRVQ5fwsnbs2DF0CQAAZAuF5NbaRI9XVRM/JgAA00G7BQAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOmOF5Kq6qaoer6oTVfX+l9jm26vqWFU9UlW/uWL556vqodG6hUkVDgAAG2Xb+TaoqrkkH0ry1iQnk9xfVfe21h5dsc1rk/ynJDe11r5QVV/ZHWZPa+2pyZUNAAAbZ5yZ5BuSnGitPdFaezbJnUne0W3zz5N8vLX2hSRprT052TIBAODiGSckX5XkiysenxwtW+lvJ9lRVb9RVUer6vtXrGtJPjVafutL/ZCqurWqFqpqYXFxcdz6AQBg4s7bbpGkVlnWVjnO303yliSvSPI/q+p3Wmv/K8mbWmunRi0Yn66qz7bWPvOiA7Z2R5I7kmR+fr4/PgAAXDTjzCSfTHLNisdXJzm1yjb3tdb+YtR7/Jkk35AkrbVTo+9PJrk7y+0bAAAwtcYJyfcneUNVvb6qLknyziT3dtvck+QfVNW2qnplkm9J8lhVXVZVr06SqrosyduSPDy58gEAYPLO227RWnuuqt6b5JNJ5pJ8pLX2SFXdNlp/oLX2WFXdl+TBJGeTfLi19nBVfV2Su6vq3M/6aGvtvo16MgAAMAnV2vS1/87Pz7eFhem+pHJVZRrHDgCA8VTV0dba/Grr3HEPAAA6QjIAAHSEZAAA6AjJAADQEZIBAKAjJAMAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOhsG7qAaVNVG7Jta2095QAAMAAhuSPMAgCg3QIAADpCMgAAdIRkAADoCMkAANARkgEAoCMkAwBAR0gGAICOkAwAAB0hGQAAOkIyAAB0hGQAAOgIyQAA0BGSAQCgIyQDAEBHSAYAgI6QDAAAHSEZAAA6QjIAAHSqtTZ0DS9SVYtJ/mDoOs7jiiRPDV3EJmI8J8t4TpbxnBxjOVnGc7KM5+TMylh+bWvtytVWTGVIngVVtdBamx+6js3CeE6W8Zws4zk5xnKyjOdkGc/J2Qxjqd0CAAA6QjIAAHSE5PW7Y+gCNhnjOVnGc7KM5+QYy8kynpNlPCdn5sdSTzIAAHTMJAMAQEdIXqOquqaqDlfVY1X1SFW9b+iaZllVXVpVv1dVx0fj+RND1zTrqmquqn6/qv770LXMuqr6fFU9VFXHqmph6HpmXVW9tqruqqrPjt5D/97QNc2qqnrj6HV57utLVfWDQ9c1q6rqh0a/gx6uqkNVdenQNc2yqnrfaCwfmeXXpXaLNaqqr07y1a21B6rq1UmOJvmnrbVHBy5tJlVVJbmstfbnVbU9yZEk72ut/c7Apc2sqvo3SeaTXN5a+86h65llVfX5JPOttVm41ufUq6pfSPJbrbUPV9UlSV7ZWvu/A5c186pqLskfJvmW1tq032Ng6lTVVVn+3XN9a+0vq+qXk/xaa+3nh61sNlXV7iR3JrkhybNJ7kvyr1tr/3vQwtbBTPIatdb+qLX2wOi//1+Sx5JcNWxVs6st+/PRw+2jL3+5rVNVXZ3knyT58NC1wEpVdXmSNyc5mCSttWcF5Il5S5L/IyBfkG1JXlFV25K8MsmpgeuZZbuS/E5r7enW2nNJfjPJdw9c07oIyRegqq5L8k1JfnfgUmbaqD3gWJInk3y6tWY81+8/Jvm3Sc4OXMdm0ZJ8qqqOVtWtQxcz474uyWKS/zJqB/pwVV02dFGbxDuTHBq6iFnVWvvDJD+T5AtJ/ijJn7XWPjVsVTPt4SRvrqqvqKpXJvnHSa4ZuKZ1EZLXqapeleRjSX6wtfaloeuZZa2151tr35jk6iQ3jD6qYY2q6juTPNlaOzp0LZvIm1pr35zk5iTvqao3D13QDNuW5JuT/OfW2jcl+Ysk7x+2pNk3alt5e5JfGbqWWVVVO5K8I8nrk3xNksuq6l8MW9Xsaq09luTfJfl0llstjid5btCi1klIXodR7+zHkvxSa+3jQ9ezWYw+ev2NJDcNW8nMelOSt4/6aO9M8g+r6r8NW9Jsa62dGn1/MsndWe6xY31OJjm54pOiu7IcmrkwNyd5oLX2J0MXMsP+UZLPtdYWW2tLST6e5O8PXNNMa60dbK19c2vtzUlOJ5m5fuRESF6z0YlmB5M81lr7D0PXM+uq6sqqeu3ov1+R5Terzw5a1Ixqrf1oa+3q1tp1Wf749X+01syGrFNVXTY6OTejtoC3ZfljRNahtfbHSb5YVW8cLXpLEic8X7hbotXiQn0hybdW1StHv+PfkuXzjVinqvrK0fdrk3xPZvQ1um3oAmbQm5L8yyQPjfpok+THWmu/NlxJM+2rk/zC6OzsL0vyy601ly5jGrwuyd3LvzOzLclHW2v3DVvSzLs9yS+NWgSeSPLugeuZaaN+z7cm+VdD1zLLWmu/W1V3JXkgy20Bv59NcLe4gX2sqr4iyVKS97TWzgxd0Hq4BBwAAHS0WwAAQEdIBgCAjpAMAAAdIRkAADpCMgAAdIRkAADoCMkAANARkgEAoPP/AVXM9pF4qw7vAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import cross_val_score, RepeatedStratifiedKFold\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "def get_dataset():\n",
    "    x, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, random_state=1)\n",
    "    return x, y\n",
    "\n",
    "def get_models():\n",
    "    models = dict()\n",
    "    for i in range(2, 10):\n",
    "        rfe = RFE(estimator=DecisionTreeClassifier(), n_features_to_select=i)\n",
    "        model = DecisionTreeClassifier()\n",
    "        models[str(i)] = Pipeline(steps=[('s',rfe),('m',model)])\n",
    "    return models\n",
    "\n",
    "def evaluate_model(model, x, y):\n",
    "    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "    scores = cross_val_score(model, x, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
    "    return scores\n",
    "\n",
    "x, y = get_dataset()\n",
    "\n",
    "models = get_models()\n",
    "results, names = list(), list()\n",
    "\n",
    "for name, model in models.items():\n",
    "    scores = evaluate_model(model, x, y)\n",
    "    results.append(scores)\n",
    "    names.append(name)\n",
    "    print('>%s %.3f (%.3f)' % (name, np.mean(scores), np.std(scores)))\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.boxplot(results, labels=names, showmeans=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "355c03c5",
   "metadata": {},
   "source": [
    "### Automatically select the number of features:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "340b415f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.889 (0.028)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import RepeatedStratifiedKFold\n",
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "x, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, random_state=1)\n",
    "\n",
    "rfe = RFECV(estimator=DecisionTreeClassifier())\n",
    "model = DecisionTreeClassifier()\n",
    "pipeline = Pipeline(steps=[('s',rfe),('m',model)])\n",
    "\n",
    "cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "n_scores = cross_val_score(pipeline, x, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
    "\n",
    "print('Accuracy: %.3f (%.3f)' % (np.mean(n_scores), np.std(n_scores)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28a1871d",
   "metadata": {},
   "source": [
    "### Which features were selected:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "82379c3d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Column: 0, Selected=False, Rank: 4\n",
      "Column: 1, Selected=False, Rank: 5\n",
      "Column: 2, Selected=True, Rank: 1\n",
      "Column: 3, Selected=True, Rank: 1\n",
      "Column: 4, Selected=True, Rank: 1\n",
      "Column: 5, Selected=False, Rank: 6\n",
      "Column: 6, Selected=True, Rank: 1\n",
      "Column: 7, Selected=False, Rank: 3\n",
      "Column: 8, Selected=True, Rank: 1\n",
      "Column: 9, Selected=False, Rank: 2\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import make_classification\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "x, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, random_state=1)\n",
    "rfe = RFE(estimator=DecisionTreeClassifier(), n_features_to_select=5)\n",
    "rfe.fit(x, y)\n",
    "\n",
    "for i in range(x.shape[1]):\n",
    "    print('Column: %d, Selected=%s, Rank: %d' % (i, rfe.support_[i], rfe.ranking_[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ec94a97",
   "metadata": {},
   "source": [
    "### Explore base algorithm:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "432c1deb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHSCAYAAAAezFYoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAafUlEQVR4nO3df6zd913f8de7drJA26T2EiJoGlKhrjjz1k67ijoaDTxUKNtKBWyjERI0mEUZNEIbFCrcKUEoGhM/JFQqWZFSbWjDhXXkBxukrSqzYBhrboqdny2zUiBZEHEWiwBt15v4vT/ucXP7yXV83Fz7e+714yFZ9vl+v+fe9/U9/t7n/d7POa7uDgAA8IJXTD0AAAAsGpEMAAADkQwAAAORDAAAA5EMAAADkQwAAIPtUw+wnksvvbSvuuqqqccAAGALu//++5/u7svW27eQkXzVVVdleXl56jEAANjCqupPTrXPcgsAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYbJ96AAAA5lNVU4/wJd099QhnlUgGANgkNiJMq2rLB+5GsNwCAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAZzRXJVvb2qPlNVR6vqfevs31FVd1TVA1X1yaravWbfH1fVg1V1uKqWN3J4AAA4G7af7oCq2pbkg0neluSJJPdV1d3d/ciaw34qyeHu/q6q+sbZ8d+6Zv+e7n56A+cGAICzZp4rydckOdrdj3X3F5N8OMk7h2OuTvKJJOnuTye5qqou39BJAQDgHJknkl+b5PE1t5+YbVvrSJLvTpKquibJ1ye5Yravk3ysqu6vqhte3rgAAHD2nXa5RZJaZ1sPt382yS9V1eEkDyb5wyTPzfa9tbufrKqvSfLxqvp0d9/7oneyGtA3JMmVV1455/iLrWq9v7pzr3v8dAEA8FLmieQnkrxuze0rkjy59oDufjbJ9UlSq2X42dmvdPeTs9+fqqo7srp840WR3N23JbktSZaWlrZE1W1EnFaVyAUAOMfmWW5xX5I3VNXrq+rCJO9KcvfaA6rqNbN9SfJDSe7t7mer6pVV9erZMa9M8m1JHtq48QEAYOOd9kpydz9XVe9J8tEk25J8qLsfrqobZ/v3J9mV5Feq6vkkjyTZO7v75UnumC072J7kV7v7no3/MAAAYOPUIv4of2lpqZeXvaRyYrkFALCxtMULqur+7l5ab5//cQ8AAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAG26ceAICkqqYeIUnS3VOPALAQRDLAAni5cVpVAhdgA1luAQAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAACD7VMPAOebqpp6hCRJd089AnAazhcwHZEM59jL/WJTVb5gwXnC+QKmY7kFAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMtk89wKLauXNnjh8/PvUYSZKqmvT979ixI88888ykM8AiW5TzxdTnisT5Atg6RPIpHD9+PN099RgLYRG+8MIic754gfMFsFVYbgEAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAg7kiuareXlWfqaqjVfW+dfbvqKo7quqBqvpkVe2e974AALBoThvJVbUtyQeTfEeSq5NcV1VXD4f9VJLD3f13k3x/kl86g/sCAMBCmedK8jVJjnb3Y939xSQfTvLO4Zirk3wiSbr700muqqrL57wvAAAslHki+bVJHl9z+4nZtrWOJPnuJKmqa5J8fZIr5rwvAAAslO1zHFPrbOvh9s8m+aWqOpzkwSR/mOS5Oe+7+k6qbkhyQ5JceeWVc4wFALC57Ny5M8ePH596jFStl2jn1o4dO/LMM89MPcYpzRPJTyR53ZrbVyR5cu0B3f1skuuTpFb/1j87+/XVp7vvmrdxW5LbkmRpaWndkAYA2MyOHz+ebpmTLEaov5R5llvcl+QNVfX6qrowybuS3L32gKp6zWxfkvxQkntn4Xza+wIAwKI57ZXk7n6uqt6T5KNJtiX5UHc/XFU3zvbvT7Irya9U1fNJHkmy96Xue3Y+FAAA2Bi1iJf8l5aWenl5edIZqsqPQ2b8XSwWn4/F43PyAn8Xi8XnY/H4nLxgEf4uqur+7l5ab5//cQ8AAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYieYEd+9yxvPued+fpzz899SgAAOcVkbzA9j+wP5/6809l/5H9U48CAHBe2T71AKzv2OeO5a6jd6XTufPonbnxTTfm0q+6dOqxgHX0zRcnt1wy9RgLoW++eOoRYKE5X7xg0c8XInlB7X9gf070iSTJiT6R/Uf25/1vef/EUwHrqZ9+Nt099RgLoarSt0w9BSwu54sXLPr5wnKLBXTyKvLKiZUkycqJldx59E5rk4F1ef4CwMYTyQto7VXkk05eTQYYef4CMC/fVM9PJC+gI08d+dJV5JNWTqzk8FOHpxkIWFjj8xd84VscO3fuTFVN+ivJ5DNUVXbu3DnxZ4OTfFM9P2uSF9BHvvMjU48AbBKev7C4jh8/bu3pzMlgZ1peFODMuJIMsEl5/gJwJtb7pppTE8kAm5TnLwDz8k31mRPJAJuU5y8A8/JN9ZmzJhlgk/L8BWBevqk+cyIZAGCL8031mbPcAgAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAZeAu4U+uaLk1sumXqMhdA3Xzz1CAAA55RIPoX66WfT3VOPsRCqKn3L1FMAAJw7llsAAMBAJAMAwEAkAwDAQCQDAMBAJAMAwEAkAwDAQCQDAMBAJAMAwEAkAwDAQCQDAMBAJAMAwGD71AMAbAVVNfUIC2HHjh1TjwCwIUQywMvU3VOPkKpaiDkAtgrLLQAAYCCSAQBgYLnFS7DGcJU1hi/YuXNnjh8/PvUYC/HY3LFjR5555pmpxwCAs0Ikn8KirO2zznCxHD9+3OdjZhFCHQDOFsstAABgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgsH3qAQAAzidVNfUIC2HHjh1Tj/CSRDIAwDnS3VOPkKpaiDkWneUWAAAwEMkAADAQyQAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADCYK5Kr6u1V9ZmqOlpV71tn/yVV9ZtVdaSqHq6q69fs++OqerCqDlfV8kYODwAAZ8P20x1QVduSfDDJ25I8keS+qrq7ux9Zc9iPJHmku99RVZcl+UxV/efu/uJs/57ufnqjhwcAgLNhnivJ1yQ52t2PzaL3w0neORzTSV5dVZXkVUmeSfLchk4KAMzt2OeO5d33vDtPf941KvhKzBPJr03y+JrbT8y2rfXLSXYleTLJg0l+tLtPzPZ1ko9V1f1VdcPLnBcAmMP+B/bnU3/+qew/sn/qUWBTOu1yiyS1zrYebn97ksNJ/lGSb0jy8ar63e5+Nslbu/vJqvqa2fZPd/e9L3onqwF9Q5JceeWVZ/AhwLnTN1+c3HLJ1GMshL754qlHgIU25fni2LZX5K4rvi79ilfkzkcP5MaP/0Iuff7E6e94ljhfsBnNE8lPJHndmttXZPWK8VrXJ/nZ7u4kR6vqs0m+Mcknu/vJJOnup6rqjqwu33hRJHf3bUluS5KlpaUxwmEh1E8/m9WH+TSOfe5Y3nvve/Pz3/zzufSrLp1sjiSpqvQtk44AC23K88X+P/iZnPjfdyQnVnJi+9/I/rf9WN7/lvdPMkvifMHmNM9yi/uSvKGqXl9VFyZ5V5K7h2P+NMm3JklVXZ7kjUkeq6pXVtWrZ9tfmeTbkjy0UcPD+caPT4HTOfa5Y7nr6F1ZObGSJFk5sZI7j95pbTKcodNGcnc/l+Q9ST6a5NEkv97dD1fVjVV14+ywn0nyTVX1YJJPJPnJ2atZXJ7kUFUdSfLJJP+9u+85Gx8IbHUnv/B12hc84JT2P7A/J/rLl1ac6BO+uYYzNM9yi3T3byX5rWHb/jV/fjKrV4nH+z2W5E0vc0YgX/6F7+QXvCl/fAospiNPHfnSVeSTVk6s5PBTh6cZCDapuSIZmNapfnx645tunHxtMrBYPvKdH5l6BNgS/LfUsAn48SkAnFsiGTYBPz4FgHPLcgvYBPz4FADOLVeSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgsH3qAbayqlqIt9PdGzIHADCtRWmLZOv3hUg+i7b6gwcAOLe0xbljuQUAAAxEMgAADEQyAAAMRDIAAAxEMgAADEQyAAAMRDIAAAy8TjKcoY16IffNbseOHVOPAABnjUiGM7AIL+JeVQsxBwBsZZZbAADAQCQDAMBAJAMAwEAkAwDAQCQDAMBAJAMAwMBLwAEsgI14/e2NeBteXnBjeV31VV5Xnc1IJAMsAHG69SzC59TrqsNXznILAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGSATezAgQPZvXt3tm3blt27d+fAgQNTjwSwJWyfegAAvjIHDhzIvn37cvvtt+faa6/NoUOHsnfv3iTJddddN/F0AJubK8kAm9Stt96a22+/PXv27MkFF1yQPXv25Pbbb8+tt9469WgAm15199QzvMjS0lIvLy9PPQYspKrKIv675dzbtm1bvvCFL+SCCy740raVlZVcdNFFef755yecjEXhfAEvraru7+6l9fa5kgywSe3atSuHDh36sm2HDh3Krl27JpoIYOsQyQCb1L59+7J3794cPHgwKysrOXjwYPbu3Zt9+/ZNPRrApueJewCb1Mkn591000159NFHs2vXrtx6662etAewAaxJhk3GGkNgXs4X8NKsSQYAgDMgkgEAYCCSAQBgIJIBAGAgkgEAYCCSAQBgIJIBAGAgkgEAYDBXJFfV26vqM1V1tKret87+S6rqN6vqSFU9XFXXz3tfAABYNKeN5KraluSDSb4jydVJrquqq4fDfiTJI939piTfkuQXqurCOe8LAAALZZ4rydckOdrdj3X3F5N8OMk7h2M6yaurqpK8KskzSZ6b874AALBQ5onk1yZ5fM3tJ2bb1vrlJLuSPJnkwSQ/2t0n5rwvAAAslHkiudbZ1sPtb09yOMnXJXlzkl+uqovnvO/qO6m6oaqWq2r52LFjc4wFAABnxzyR/ESS1625fUVWrxivdX2S3+hVR5N8Nsk3znnfJEl339bdS929dNlll807PwAAbLh5Ivm+JG+oqtdX1YVJ3pXk7uGYP03yrUlSVZcneWOSx+a8LwAALJTtpzugu5+rqvck+WiSbUk+1N0PV9WNs/37k/xMkv9QVQ9mdYnFT3b300my3n3PzocCAAAbo7rXXSI8qaWlpV5eXp56DFhIVZVF/HcLLB7nC3hpVXV/dy+tt8//uAcAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAA5EMAAADkQwAAAORDAAAg+1TDwDnm6paiLfR3S/7bQDAViWS4RwTpwCw+Cy3AACAgUgGAICBSAYAgIFIBgCAgUgGAICBSAYAgIFIBgCAgUgGAICBSAYAgIFIBgCAgUgGAICBSAYAgIFIBgCAgUgGAICBSAYAgMH2qQcAANZXVQvxNrr7Zb8N2GxEMgAsKHEK07HcAgAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGc0VyVb29qj5TVUer6n3r7H9vVR2e/Xqoqp6vqp2zfX9cVQ/O9i1v9AcAAAAbbfvpDqiqbUk+mORtSZ5Icl9V3d3dj5w8prt/LsnPzY5/R5J/3d3PrHkze7r76Q2dHAAAzpJ5riRfk+Rodz/W3V9M8uEk73yJ469LcmAjhgMAgCnME8mvTfL4mttPzLa9SFV9dZK3J/mvazZ3ko9V1f1VdcNXOigAAJwrp11ukaTW2danOPYdSX5vWGrx1u5+sqq+JsnHq+rT3X3vi97JakDfkCRXXnnlHGMBAMDZMc+V5CeSvG7N7SuSPHmKY9+VYalFdz85+/2pJHdkdfnGi3T3bd291N1Ll1122RxjAQDA2TFPJN+X5A1V9fqqujCrIXz3eFBVXZLkm5PctWbbK6vq1Sf/nOTbkjy0EYMDAMDZctrlFt39XFW9J8lHk2xL8qHufriqbpzt3z879LuSfKy7/3rN3S9PckdVnXxfv9rd92zkBwAAAButuk+1vHg6S0tLvbzsJZUBADh7qur+7l5ab5//cQ8AAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZAAAGIhkAAAYiGQAABiIZNokDBw5k9+7d2bZtW3bv3p0DBw5MPRIAbFnbpx4AOL0DBw5k3759uf3223Pttdfm0KFD2bt3b5Lkuuuum3g6ANh6qrunnuFFlpaWenl5eeoxYGHs3r07H/jAB7Jnz54vbTt48GBuuummPPTQQxNOBgCbV1Xd391L6+4TybD4tm3bli984Qu54IILvrRtZWUlF110UZ5//vkJJwOAzeulItmaZNgEdu3alUOHDn3ZtkOHDmXXrl0TTQQAW5tIhk1g37592bt3bw4ePJiVlZUcPHgwe/fuzb59+6YeDQC2JE/cg03g5JPzbrrppjz66KPZtWtXbr31Vk/aA4CzxJpkAADOS9YkAwDAGRDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADAQyQAAMBDJAAAwEMkAADCo7p56hhepqmNJ/mTqORbEpUmennoIForHBOvxuGA9Hhesx+PiBV/f3Zett2MhI5kXVNVydy9NPQeLw2OC9XhcsB6PC9bjcTEfyy0AAGAgkgEAYCCSF99tUw/AwvGYYD0eF6zH44L1eFzMwZpkAAAYuJIMAAADkbyAquqvpp4B2Bqq6s1V9Y+nnoPpVNU/r6pHq+rg1LMwjar6naryahZnSCRvElW1beoZ2ByqavvUM7AYZo+FNycRyeepqqok/zLJD3f3nqnngc3EmuQFVFV/1d2vqqpvSXJzkj9L8ubuvnrSwThnquqqJPck+V9J/l6SP0ry/Ul2JfnFJK/K6gvBv7u7/6yqfifJ7yd5a5K7u/sXJhibs6iqvj/JjyfpJA8k+fUk709yYZL/m+T7uvvPq+qWJF+X5KqsPkauTfJVSf5Pkn/X3b92zofnnJqdP347ycEk/2q2+Y+yem5471RzcW5U1b9N8n1JHs/qOeD+JP80yeEk1yS5OMkPdvcnZ+eL1yf52iR/K8m/SfKWJN+R1XPGO7p75Rx/CAvDFafFd02S3d392akH4Zx7Y5K93f17VfWhJD+S5LuSvLO7j1XV9ya5NckPzo5/TXd/80SzchZV1d9Osi/JW7v76aramdVYfkt3d1X9UJKfSPJjs7v8/STXdvfnq+rdSZa6+z1TzM5k3pjk+u7+4dk30T/e3csTz8RZNltS8T1ZvbiyPcmnshrJSfLK7v6mqvqHST6UZPds+zck2ZPk6iT/M8n3dPdPVNUdSf5JkjvP3UewWETy4vukQD5vPd7dvzf7839K8lNZPal9fPUnqNmW1Z8ynOQK4db1j5J8pLufTpLufqaq/k6SX6uqr83q1eS154m7u/vzE8zJ4viT7v6DqYfgnLs2yV0n//1X1W+u2XcgSbr73qq6uKpeM9v+2929UlUPZvXryj2z7Q9m9SdS5y2RvPj+euoBmMy4Fuovkzzc3f/gFMd7rGxdlRc/Hj6Q5Be7++7Z0qxb1uzzWMBj4PxUL7FvPIecvP3/kqS7T1TVSr+wDvdEzvNO9MQ9WFxXVtXJIL4uyR8kuezktqq6YPZjeLa+TyT5F1X1N5NkttzikqyuGUySH3iJ+/5lklef3fGABXEoyTuq6qKqelVWl0uc9L1JUlXXJvmL7v6LKQbcTEQyLK5Hk/xAVT2QZGdWrxz+syT/vqqOZPVJGN803XicK939cFbXn/+P2ef+F7N65fi/VNXvZvXJOadyMMnVVXV4to4d2KK6+74kdyc5kuQ3kiwnORnDx6vq95PsT7J3mgk3F69uAQto9uz0/9bdu093LACcVFWv6u6/qqqvTnJvkhu6+1NTz7UZnddrTQAAtpjbqurqJBcl+Y8C+SvnSjIAAAysSQYAgIFIBgCAgUgGAICBSAYAgIFIBgCAgUgGAIDB/wfmDO4AF+jJWQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import cross_val_score, RepeatedStratifiedKFold\n",
    "from sklearn.feature_selection import RFE\n",
    "from sklearn.linear_model import LogisticRegression, Perceptron\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "\n",
    "def get_dataset():\n",
    "    x, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, random_state=1)\n",
    "    return x, y\n",
    "\n",
    "def get_models():\n",
    "    models = dict()\n",
    "    \n",
    "    rfe = RFE(estimator=LogisticRegression(), n_features_to_select=5)\n",
    "    model = DecisionTreeClassifier()\n",
    "    models['lr'] = Pipeline(steps=[('s',rfe),('m',model)])\n",
    "\n",
    "    rfe = RFE(estimator=Perceptron(), n_features_to_select=5)\n",
    "    model = DecisionTreeClassifier()\n",
    "    models['per'] = Pipeline(steps=[('s',rfe),('m',model)])\n",
    "    \n",
    "    rfe = RFE(estimator=DecisionTreeClassifier(), n_features_to_select=5)\n",
    "    model = DecisionTreeClassifier()\n",
    "    models['cart'] = Pipeline(steps=[('s',rfe),('m',model)])\n",
    "    \n",
    "    rfe = RFE(estimator=RandomForestClassifier(), n_features_to_select=5)\n",
    "    model = DecisionTreeClassifier()\n",
    "    models['rf'] = Pipeline(steps=[('s',rfe),('m',model)])\n",
    "    \n",
    "    rfe = RFE(estimator=GradientBoostingClassifier(), n_features_to_select=5)\n",
    "    model = DecisionTreeClassifier()\n",
    "    models['gbm'] = Pipeline(steps=[('s',rfe),('m',model)])\n",
    "    return models\n",
    "\n",
    "def evaluate_model(model, X, y):\n",
    "    cv = RepeatedStratifiedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
    "    scores = cross_val_score(model, x, y, scoring='accuracy', cv=cv, n_jobs=-1)\n",
    "    return scores\n",
    "\n",
    "x, y = get_dataset()\n",
    "\n",
    "models = get_models()\n",
    "\n",
    "results, names = list(), list()\n",
    "\n",
    "for name, model in models.items():\n",
    "    scores = evaluate_model(model, x, y)\n",
    "    results.append(scores)\n",
    "    names.append(name)\n",
    "    \n",
    "plt.figure(figsize=(12,8))\n",
    "plt.boxplot(results, labels=names, showmeans=True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d68a05e4",
   "metadata": {},
   "source": [
    "# Feature Importance:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf7c0c8f",
   "metadata": {},
   "source": [
    "## Linear Regression Feature Importance:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "a8b20db8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature: 0, Score: -0.00000\n",
      "Feature: 1, Score: 12.44483\n",
      "Feature: 2, Score: 0.00000\n",
      "Feature: 3, Score: -0.00000\n",
      "Feature: 4, Score: 93.32225\n",
      "Feature: 5, Score: 86.50811\n",
      "Feature: 6, Score: 26.74607\n",
      "Feature: 7, Score: 3.28535\n",
      "Feature: 8, Score: 0.00000\n",
      "Feature: 9, Score: 0.00000\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA54AAAHSCAYAAACEihTZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAATLUlEQVR4nO3dT6idB5nH8d8zvYq2IrY0LbUtkwpFLUKpBKkWXEyU0VFsN4UIShCHbvxTxSLVjVsXQXQhQqhKwGKQWmgRUUvUxWyKaWvRGktLnWmjsb0O+AcXU4vPLO7RSTuV3Cb3yTk39/PZnPu+99zc3+KQ5Jv3vSfV3QEAAIAp/7TsAQAAAJzbhCcAAACjhCcAAACjhCcAAACjhCcAAACjhCcAAACj1s7mN7v44ot79+7dZ/NbAgAAcJY88MADv+vuXS88f1bDc/fu3Tl69OjZ/JYAAACcJVX1Xy923q22AAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjBKeAAAAjFpb9gAAtp9rD+xb9oSV8PBth5c9AQC2BVc8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGCU8AQAAGLW27AEAsJNde2DfsieshIdvO7zsCQAMcsUTAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUcITAACAUZsKz6r6ZFU9UlU/r6pvVtUrquqiqrqvqh5bPF44PRYAAIDt55ThWVWXJ/l4kj3d/aYk5yXZl+T2JEe6++okRxbHAAAA8DybvdV2Lckrq2otyflJfpPkxiSHFp8/lOSmLV8HAADAtnfK8OzuXyc5kOTJJCeS/KG7f5Dk0u4+sXjOiSSXTA4FAABge9rMrbYXZuPq5lVJXpvkgqr6wGa/QVXdUlVHq+ro+vr66S8FAABgW9rMrbbvSPKr7l7v7r8kuTvJ25I8XVWXJcni8ZkX++LuPtjde7p7z65du7ZqNwAAANvEZsLzySTXV9X5VVVJ9iY5luTeJPsXz9mf5J6ZiQAAAGxna6d6QnffX1V3JXkwyXNJHkpyMMmrknyrqj6cjTi9eXIoAAAA29MpwzNJuvtzST73gtP/k42rnwAAAPAPbfa/UwEAAIDTIjwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYtanwrKrXVNVdVfXLqjpWVW+tqouq6r6qemzxeOH0WAAAALafzV7x/FKS73X3G5Jcm+RYktuTHOnuq5McWRwDAADA85wyPKvq1UnenuSrSdLdz3b375PcmOTQ4mmHktw0MxEAAIDtbDNXPF+XZD3J16vqoaq6o6ouSHJpd59IksXjJYM7AQAA2KY2E55rSd6c5CvdfV2SP+cl3FZbVbdU1dGqOrq+vn6aMwEAANiuNhOex5Mc7+77F8d3ZSNEn66qy5Jk8fjMi31xdx/s7j3dvWfXrl1bsRkAAIBt5JTh2d2/TfJUVb1+cWpvkl8kuTfJ/sW5/UnuGVkIAADAtra2yed9LMmdVfXyJE8k+VA2ovVbVfXhJE8muXlmIgAAANvZpsKzu3+aZM+LfGrvlq4BAADgnLPZ/8cTAAAATovwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYJTwBAAAYNSmw7Oqzquqh6rqO4vji6rqvqp6bPF44dxMAAAAtquXcsXz1iTHTjq+PcmR7r46yZHFMQAAADzPpsKzqq5I8p4kd5x0+sYkhxYfH0py05YuAwAA4Jyw2SueX0zy6SR/Pencpd19IkkWj5e82BdW1S1VdbSqjq6vr5/JVgAAALahU4ZnVb03yTPd/cDpfIPuPtjde7p7z65du07nlwAAAGAbW9vEc25I8r6q+rckr0jy6qr6RpKnq+qy7j5RVZcleWZyKAAAANvTKa94dvdnuvuK7t6dZF+SH3b3B5Lcm2T/4mn7k9wzthIAAIBt60z+H8/PJ3lnVT2W5J2LYwAAAHiezdxq+3fd/eMkP158/N9J9m79JAAAAM4lZ3LFEwAAAE5JeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBqbdkDAADO1LUH9i17wsp4+LbDy54A8P+44gkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMAo4QkAAMCoU4ZnVV1ZVT+qqmNV9UhV3bo4f1FV3VdVjy0eL5yfCwAAwHazmSuezyX5VHe/Mcn1ST5SVdckuT3Jke6+OsmRxTEAAAA8zynDs7tPdPeDi4//lORYksuT3Jjk0OJph5LcNLQRAACAbewl/YxnVe1Ocl2S+5Nc2t0nko04TXLJlq8DAABg29t0eFbVq5J8O8knuvuPL+Hrbqmqo1V1dH19/XQ2AgAAsI1tKjyr6mXZiM47u/vuxemnq+qyxecvS/LMi31tdx/s7j3dvWfXrl1bsRkAAIBtZDPvaltJvprkWHd/4aRP3Ztk/+Lj/Unu2fp5AAAAbHdrm3jODUk+mORnVfXTxbnPJvl8km9V1YeTPJnk5pGFAAAAbGunDM/u/o8k9Q8+vXdr5wAAAHCueUnvagsAAAAvlfAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABg1NqyB3BuuvbAvmVPWBkP33Z42RMAAGCpXPEEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABglPAEAABg1NqyBwAAsDquPbBv2RNWxsO3HV72BDhnuOIJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAKOEJAADAqDMKz6p6V1U9WlWPV9XtWzUKAACAc8dph2dVnZfky0neneSaJO+vqmu2ahgAAADnhjO54vmWJI939xPd/WySw0lu3JpZAAAAnCvOJDwvT/LUScfHF+cAAADg76q7T+8Lq25O8q/d/e+L4w8meUt3f+wFz7slyS2Lw9cnefT05+4YFyf53bJHwEm8JllFXpesIq9LVo3XJGfbP3f3rheeXDuDX/B4kitPOr4iyW9e+KTuPpjk4Bl8nx2nqo52955l74C/8ZpkFXldsoq8Llk1XpOsijO51fYnSa6uqquq6uVJ9iW5d2tmAQAAcK447Sue3f1cVX00yfeTnJfka939yJYtAwAA4JxwJrfapru/m+S7W7SF/+PWZFaN1ySryOuSVeR1yarxmmQlnPabCwEAAMBmnMnPeAIAAMApCc8VUlXvqqpHq+rxqrp92Xugqq6sqh9V1bGqeqSqbl32JkiSqjqvqh6qqu8sewskSVW9pqruqqpfLn7PfOuyN0FVfXLx5/fPq+qbVfWKZW9i5xKeK6Kqzkvy5STvTnJNkvdX1TXLXQV5LsmnuvuNSa5P8hGvS1bErUmOLXsEnORLSb7X3W9Icm28Plmyqro8yceT7OnuN2XjzUD3LXcVO5nwXB1vSfJ4dz/R3c8mOZzkxiVvYofr7hPd/eDi4z9l4y9Sly93FTtdVV2R5D1J7lj2FkiSqnp1krcn+WqSdPez3f37pY6CDWtJXllVa0nOT/KbJe9hBxOeq+PyJE+ddHw8/oLPCqmq3UmuS3L/kqfAF5N8Oslfl7wD/uZ1SdaTfH1xC/gdVXXBskexs3X3r5McSPJkkhNJ/tDdP1juKnYy4bk66kXOecthVkJVvSrJt5N8orv/uOw97FxV9d4kz3T3A8veAidZS/LmJF/p7uuS/DmJ92pgqarqwmzcPXdVktcmuaCqPrDcVexkwnN1HE9y5UnHV8TtEKyAqnpZNqLzzu6+e9l72PFuSPK+qvrPbPxIwr9U1TeWOwlyPMnx7v7bHSF3ZSNEYZnekeRX3b3e3X9JcneSty15EzuY8FwdP0lydVVdVVUvz8YPf9+75E3scFVV2fiZpWPd/YVl74Hu/kx3X9Hdu7Px++QPu9u/4LNU3f3bJE9V1esXp/Ym+cUSJ0GycYvt9VV1/uLP873xplcs0dqyB7Chu5+rqo8m+X423nXsa939yJJnwQ1JPpjkZ1X108W5z3b3d5c3CWAlfSzJnYt/PH4iyYeWvIcdrrvvr6q7kjyYjXepfyjJweWuYierbj9GCAAAwBy32gIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADBKeAIAADDqfwFPFXtBEjwEigAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=10, n_informative=5, random_state=1)\n",
    "\n",
    "model = LinearRegression()\n",
    "model.fit(x, y)\n",
    "\n",
    "importance = model.coef_\n",
    "\n",
    "for i,v in enumerate(importance):\n",
    "    print('Feature: %0d, Score: %.5f' % (i,v))\n",
    "\n",
    "plt.figure(figsize=(16,8))\n",
    "\n",
    "plt.bar([x for x in range(len(importance))], importance, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4900203",
   "metadata": {},
   "source": [
    "## Logistic Regression Feature Importance:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "b138ef92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature: 0, Score: 0.15174\n",
      "Feature: 1, Score: -0.56581\n",
      "Feature: 2, Score: -0.47250\n",
      "Feature: 3, Score: -0.14069\n",
      "Feature: 4, Score: 0.06982\n",
      "Feature: 5, Score: -0.74609\n",
      "Feature: 6, Score: 0.12295\n",
      "Feature: 7, Score: 0.04651\n",
      "Feature: 8, Score: -0.24610\n",
      "Feature: 9, Score: -0.22746\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAssAAAHSCAYAAAAaOYYVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAARe0lEQVR4nO3dX4jld3nH8c/TjGJVxEii5t92FYIaCmlkkdiAFJOARjFeRtAGb0JBbSwuEut98SKIFkQJUUkxdC+iYJCg1agXvWgwMa41rpJgq9lmNVHwD/YiDT69mFMI23l2k5w581t3Xi8Y5vz5ku8z/DKzb37zm3OquwMAAPx/f7L0AAAAcKYSywAAMBDLAAAwEMsAADAQywAAMBDLAAAw2Fp6gFM577zz+uDBg0uPAQDAWeyBBx74ZXefv9NzZ3QsHzx4MPfff//SYwAAcBarqp9Oz7kMAwAABmIZAAAGYhkAAAZiGQAABmIZAAAGYhkAAAZiGQAABmIZAAAGYhkAAAZiGQAABmIZAAAGYhkAAAZiGQAABmIZAAAGYhkAAAZiGQAABmIZAAAGYhkAAAZbSw9wJrr81huWHmFXHD18ZOkRAAD+qDmzDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAACDraUHAIBNuvzWG5YeYW1HDx9ZegTYt5xZBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAIDB1tIDAOyly2+9YekRdsXRw0eWHgFgX3BmGQAABmIZAAAGYhkAAAZiGQAABmIZAAAGYhkAAAZiGQAABmIZAAAGYhkAAAbewQ8AzkLerRJ2hzPLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAw2JVYrqq3VNWPq+qRqrplh+erqv5x9fz3q+r1u7EvAABs0tqxXFXnJPlUkrcmuSzJu6rqspOWvTXJpauPm5J8et19AQBg03bjzPIbkjzS3T/p7ieTHEly/Ulrrk/yT73t35K8tKou2IW9AQBgY3Yjli9K8ujT7h9fPfZs1wAAwBllaxf+G7XDY/0c1mwvrLop25dq5MCBA+tN9hwdPXxkkX2XdvmtNyw9wq54tsfP1/3H7dl+3fv1+zs5O475czl++/WY79ev+2z4/zzxM/1Mshtnlo8nueRp9y9O8thzWJMk6e7buvtQdx86//zzd2E8AAB4bnYjlr+T5NKqelVVPT/JDUnuPmnN3Un+evWqGFcm+U13n9iFvQEAYGPWvgyju5+qqvcn+VqSc5J8rrsfqqq/WT3/mST3JLkuySNJ/jvJe9fdFwAANm03rllOd9+T7SB++mOfedrtTvK+3dgLAAD2infwAwCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAICBWAYAgIFYBgCAgVgGAIDB1tIDAMs4evjI0iMAwBnPmWUAABiIZQAAGIhlAAAYiGUAABiIZQAAGIhlAAAYiGUAABiIZQAAGIhlAAAYiGUAABiIZQAAGIhlAAAYiGUAABiIZQAAGIhlAAAYiGUAABiIZQAAGIhlAAAYiGUAABiIZQAAGIhlAAAYiGUAABisFctV9bKq+npVPbz6fO4Oay6pqm9V1bGqeqiqbl5nTwAA2Cvrnlm+Jcm93X1pkntX90/2VJIPdffrklyZ5H1Vddma+wIAwMatG8vXJ7ljdfuOJO88eUF3n+ju765u/y7JsSQXrbkvAABs3Lqx/IruPpFsR3GSl59qcVUdTHJFkvvW3BcAADZu63QLquobSV65w1MffTYbVdWLk3wxyQe7+7enWHdTkpuS5MCBA89mCwAA2FWnjeXuvmZ6rqp+UVUXdPeJqrogyePDuudlO5Tv7O4vnWa/25LcliSHDh3q080HAACbsu5lGHcnuXF1+8YkXz55QVVVks8mOdbdH19zPwAA2DPrxvLHklxbVQ8nuXZ1P1V1YVXds1pzVZL3JHlzVX1v9XHdmvsCAMDGnfYyjFPp7l8luXqHxx9Lct3q9r8mqXX2AQCAJXgHPwAAGIhlAAAYiGUAABiIZQAAGIhlAAAYiGUAABiIZQAAGKz1OssAACzv6OEjS49w1nJmGQAABmIZAAAGYhkAAAZiGQAABmIZAAAGYhkAAAZiGQAABl5nGQA4a3i9YXabM8sAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwWCuWq+plVfX1qnp49fncU6w9p6oerKqvrLMnAADslXXPLN+S5N7uvjTJvav7k5uTHFtzPwAA2DPrxvL1Se5Y3b4jyTt3WlRVFyd5W5Lb19wPAAD2zLqx/IruPpEkq88vH9Z9IsmHk/xhzf0AAGDPbJ1uQVV9I8krd3jqo89kg6p6e5LHu/uBqvqrZ7D+piQ3JcmBAweeyRYAALARp43l7r5meq6qflFVF3T3iaq6IMnjOyy7Ksk7quq6JC9I8pKq+kJ3v3vY77YktyXJoUOH+pl8EQAAsAnrXoZxd5IbV7dvTPLlkxd090e6++LuPpjkhiTfnEIZAADOJOvG8seSXFtVDye5dnU/VXVhVd2z7nAAALCk016GcSrd/askV+/w+GNJrtvh8W8n+fY6ewIAwF7xDn4AADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAwEMsAADAQywAAMBDLAAAw2Fp6AFja0cNHlh4BADhDObMMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAACDraUH4Mxx9PCRpUcAADijOLMMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAAADsQwAAAOxDAAAA7EMAACDtWK5ql5WVV+vqodXn88d1r20qu6qqh9V1bGqeuM6+wIAwF5Y98zyLUnu7e5Lk9y7ur+TTyb5ane/NsnlSY6tuS8AAGzcurF8fZI7VrfvSPLOkxdU1UuSvCnJZ5Oku5/s7l+vuS8AAGzcurH8iu4+kSSrzy/fYc2rkzyR5PNV9WBV3V5VL1pzXwAA2LjTxnJVfaOqfrDDx/XPcI+tJK9P8unuviLJ7zNfrpGquqmq7q+q+5944olnuAUAAOy+rdMt6O5rpueq6hdVdUF3n6iqC5I8vsOy40mOd/d9q/t35RSx3N23JbktSQ4dOtSnmw8AADZl3csw7k5y4+r2jUm+fPKC7v55kker6jWrh65O8sM19wUAgI1bN5Y/luTaqno4ybWr+6mqC6vqnqet+0CSO6vq+0n+Isk/rLkvAABs3GkvwziV7v5Vts8Un/z4Y0mue9r97yU5tM5eAACw17yDHwAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMtpYeAIC9cfTwkaVHAPij48wyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMxDIAAAzEMgAADMQyAAAMqruXnmFUVU8k+enSc2zIeUl+ufQQ7BnHe39xvPcfx3x/cbzPPn/W3efv9MQZHctns6q6v7sPLT0He8Px3l8c7/3HMd9fHO/9xWUYAAAwEMsAADAQy8u5bekB2FOO9/7ieO8/jvn+4njvI65ZBgCAgTPLAAAwEMt7rKreUlU/rqpHquqWpedhs6rqkqr6VlUdq6qHqurmpWdi86rqnKp6sKq+svQsbFZVvbSq7qqqH62+z9+49ExsTlX93epn+Q+q6p+r6gVLz8TmieU9VFXnJPlUkrcmuSzJu6rqsmWnYsOeSvKh7n5dkiuTvM8x3xduTnJs6SHYE59M8tXufm2Sy+O4n7Wq6qIkf5vkUHf/eZJzktyw7FTsBbG8t96Q5JHu/kl3P5nkSJLrF56JDeruE9393dXt32X7H9KLlp2KTaqqi5O8LcntS8/CZlXVS5K8Kclnk6S7n+zuXy86FJu2leRPq2oryQuTPLbwPOwBsby3Lkry6NPuH49w2jeq6mCSK5Lct/AobNYnknw4yR8WnoPNe3WSJ5J8fnXZze1V9aKlh2Izuvu/ktya5GdJTiT5TXf/y7JTsRfE8t6qHR7zciT7QFW9OMkXk3ywu3+79DxsRlW9Pcnj3f3A0rOwJ7aSvD7Jp7v7iiS/T+JvUc5SVXVutn8b/KokFyZ5UVW9e9mp2AtieW8dT3LJ0+5fHL/COetV1fOyHcp3dveXlp6HjboqyTuq6j+zfZnVm6vqC8uOxAYdT3K8u//vt0V3ZTueOTtdk+Q/uvuJ7v6fJF9K8pcLz8QeEMt76ztJLq2qV1XV87P9hwF3LzwTG1RVle3rGY9198eXnofN6u6PdPfF3X0w29/f3+xuZ57OUt398ySPVtVrVg9dneSHC47EZv0syZVV9cLVz/ar4w8694WtpQfYT7r7qap6f5KvZfuvaD/X3Q8tPBabdVWS9yT596r63uqxv+/ue5YbCdhFH0hy5+oEyE+SvHfhediQ7r6vqu5K8t1sv9LRg/FOfvuCd/ADAICByzAAAGAglgEAYCCWAQBgIJYBAGAglgEAYCCWAQBgIJYBAGAglgEAYPC/ezCTawPAyP8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "x, y = make_classification(n_samples=1000, n_features=10, n_informative=5, random_state=1)\n",
    "\n",
    "model = LogisticRegression()\n",
    "model.fit(x, y)\n",
    "\n",
    "importance = model.coef_[0]\n",
    "\n",
    "for i,v in enumerate(importance):\n",
    "    print('Feature: %0d, Score: %.5f' % (i,v))\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.bar([x for x in range(len(importance))], importance, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad89641e",
   "metadata": {},
   "source": [
    "## Decision Tree Feature Importance:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "825f200a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature: 0, Score: 0.00227\n",
      "Feature: 1, Score: 0.00449\n",
      "Feature: 2, Score: 0.00269\n",
      "Feature: 3, Score: 0.00214\n",
      "Feature: 4, Score: 0.51635\n",
      "Feature: 5, Score: 0.43835\n",
      "Feature: 6, Score: 0.02734\n",
      "Feature: 7, Score: 0.00282\n",
      "Feature: 8, Score: 0.00243\n",
      "Feature: 9, Score: 0.00112\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsIAAAHSCAYAAADmLK3fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAATIElEQVR4nO3df4hl91nH8c/jrkGtiGIWqruJGzVYozQoa2yt+LuYVDEVK27VFn+UJdL0hxhs9I/+0z+0EETB6LLUKKK4SK2y6NYIKqjUlt3ERk1qZIk/MqbSbdXWajFd+/jH3JRxOsneJPfmJvO8XrBwzzlf7jzLYSbvnD1zT3V3AABgmk/b9AAAALAJQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGOngpr7wlVde2UePHt3UlwcAYIh77rnng919aPf+jYXw0aNHc/78+U19eQAAhqiqf9prv1sjAAAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGCkg5seAGCVrr/j+KZHWIn7bju96REA9j1XhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjLRUCFfVjVX1YFVdqKrb9zj+TVX14ap67+LPm1c/KgAArM5lnyxXVQeS3JnkpUm2kpyrqjPd/cCupX/e3d+5hhkBAGDllrkifEOSC939UHc/muR0kpvXOxYAAKzXMiF8OMnDO7a3Fvt2e3FV3VdV76yqr1jJdAAAsCaXvTUiSe2xr3dt35vki7r7o1X1siS/l+TaT3mjqhNJTiTJ1Vdf/eQmBQCAFVrmivBWkqt2bB9J8sjOBd39ke7+6OL12SSfXlVX7n6j7j7V3ce6+9ihQ4eextgAAPD0LBPC55JcW1XXVNUVSY4nObNzQVU9v6pq8fqGxft+aNXDAgDAqlz21ojuvlRVtya5O8mBJHd19/1Vdcvi+Mkkr0jyY1V1KcnHkhzv7t23TwAAwLPGMvcIP3a7w9ld+07ueP2LSX5xtaMBAMD6eLIcAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGOngpgcA4Om7/o7jmx5hJe677fSmRwAGcUUYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMtFcJVdWNVPVhVF6rq9idY9zVV9b9V9YrVjQgAAKt32RCuqgNJ7kxyU5Lrkryyqq57nHVvTXL3qocEAIBVW+aK8A1JLnT3Q939aJLTSW7eY93rkvxOkg+scD4AAFiLZUL4cJKHd2xvLfZ9UlUdTvLdSU6ubjQAAFifZUK49tjXu7Z/Psmbuvt/n/CNqk5U1fmqOn/x4sUlRwQAgNU7uMSarSRX7dg+kuSRXWuOJTldVUlyZZKXVdWl7v69nYu6+1SSU0ly7Nix3TENAADPmGVC+FySa6vqmiT/kuR4ku/fuaC7r3nsdVX9WpLf3x3BAADwbHLZEO7uS1V1a7Y/DeJAkru6+/6qumVx3H3BAAA85yxzRTjdfTbJ2V379gzg7v6hpz8WAACslyfLAQAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIy0VAhX1Y1V9WBVXaiq2/c4fnNV/XVVvbeqzlfV169+VAAAWJ2Dl1tQVQeS3JnkpUm2kpyrqjPd/cCOZX+c5Ex3d1W9MMlvJ3nBOgYGAIBVWOaK8A1JLnT3Q939aJLTSW7euaC7P9rdvdh8XpIOAAA8iy0TwoeTPLxje2ux7/+pqu+uqr9L8gdJfmQ14wEAwHosE8K1x75PueLb3b/b3S9I8vIkb9nzjapOLO4hPn/x4sUnNSgAAKzSMiG8leSqHdtHkjzyeIu7+8+SfElVXbnHsVPdfay7jx06dOhJDwsAAKuyTAifS3JtVV1TVVckOZ7kzM4FVfWlVVWL11+d5IokH1r1sAAAsCqX/dSI7r5UVbcmuTvJgSR3dff9VXXL4vjJJN+T5NVV9fEkH0vyfTt+eQ4AAJ51LhvCSdLdZ5Oc3bXv5I7Xb03y1tWOBgAA6+PJcgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEZaKoSr6saqerCqLlTV7Xsc/4Gq+uvFn3dV1fWrHxUAAFbnsiFcVQeS3JnkpiTXJXllVV23a9k/JPnG7n5hkrckObXqQQEAYJWWuSJ8Q5IL3f1Qdz+a5HSSm3cu6O53dfe/LzbfneTIascEAIDVWiaEDyd5eMf21mLf4/nRJO/c60BVnaiq81V1/uLFi8tPCQAAK7ZMCNce+3rPhVXfnO0QftNex7v7VHcf6+5jhw4dWn5KAABYsYNLrNlKctWO7SNJHtm9qKpemORtSW7q7g+tZjwAAFiPZa4In0tybVVdU1VXJDme5MzOBVV1dZJ3JHlVd//96scEAIDVuuwV4e6+VFW3Jrk7yYEkd3X3/VV1y+L4ySRvTvL5SX6pqpLkUncfW9/YAADw9Cxza0S6+2ySs7v2ndzx+jVJXrPa0QAAYH08WQ4AgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgpKVCuKpurKoHq+pCVd2+x/EXVNVfVtX/VNVtqx8TAABW6+DlFlTVgSR3Jnlpkq0k56rqTHc/sGPZvyV5fZKXr2NIAABYtWWuCN+Q5EJ3P9TdjyY5neTmnQu6+wPdfS7Jx9cwIwAArNwyIXw4ycM7trcW+wAA4DlrmRCuPfb1U/liVXWiqs5X1fmLFy8+lbcAAICVWCaEt5JctWP7SJJHnsoX6+5T3X2su48dOnToqbwFAACsxDIhfC7JtVV1TVVdkeR4kjPrHQsAANbrsp8a0d2XqurWJHcnOZDkru6+v6puWRw/WVXPT3I+yeck+URVvTHJdd39kfWNDgAAT91lQzhJuvtskrO79p3c8fpfs33LBAAAPCd4shwAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMNLBTQ8AAE/V9Xcc3/QIK3Hfbac3PQKM5IowAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIx1cZlFV3ZjkF5IcSPK27v7ZXcdrcfxlSf47yQ91970rnpWn6fo7jm96hKftvttOb3oEAGCfuOwV4ao6kOTOJDcluS7JK6vqul3Lbkpy7eLPiSS/vOI5AQBgpZa5InxDkgvd/VCSVNXpJDcneWDHmpuT/Hp3d5J3V9XnVtUXdPf7Vz7xCrgyOst+ON/Jkz/nU//eMMHk7+/98Hf3c+3ZY5kQPpzk4R3bW0m+dok1h5M8K0MYAOC5xP8ArEdtX8R9ggVV35vk27v7NYvtVyW5obtft2PNHyT5me7+i8X2Hyf5ye6+Z9d7ncj2rRNJ8mVJHlzVX+RZ5sokH9z0EDyjnPNZnO9ZnO9ZnO/96Yu6+9DunctcEd5KctWO7SNJHnkKa9Ldp5KcWuJrPqdV1fnuPrbpOXjmOOezON+zON+zON+zLPPxaeeSXFtV11TVFUmOJzmza82ZJK+ubS9K8uFn6/3BAACQLHFFuLsvVdWtSe7O9sen3dXd91fVLYvjJ5OczfZHp13I9sen/fD6RgYAgKdvqc8R7u6z2Y7dnftO7njdSV672tGe0/b97R98Cud8Fud7Fud7Fud7kMv+shwAAOxHHrEMAMBIQnjFqurGqnqwqi5U1e2bnof1qaqrqupPq+p9VXV/Vb1h0zOxflV1oKr+qqp+f9OzsH6LB0S9var+bvG9/uJNz8T6VNWPL36e/21V/VZVfcamZ2K9hPAKLfk4avaPS0l+oru/PMmLkrzW+R7hDUnet+kheMb8QpI/7O4XJLk+zv2+VVWHk7w+ybHu/spsf0DAc/8pFjwhIbxan3wcdXc/muSxx1GzD3X3+7v73sXr/8z2fyAPb3Yq1qmqjiT5jiRv2/QsrF9VfU6Sb0jyK0nS3Y92939sdCjW7WCSz6yqg0k+K3s8E4H9RQiv1uM9app9rqqOJvmqJO/Z8Cis188n+ckkn9jwHDwzvjjJxSS/urgd5m1V9bxND8V6dPe/JLkjyT8neX+2n4nwR5udinUTwqtVe+zzsRz7XFV9dpLfSfLG7v7IpudhParqO5N8YPej49nXDib56iS/3N1fleS/kvjdj32qqj4v2/+Ke02SL0zyvKr6wc1OxboJ4dVa6lHT7B9V9enZjuDf7O53bHoe1uolSb6rqv4x27c9fUtV/cZmR2LNtpJsdfdj/9Lz9myHMfvTtyX5h+6+2N0fT/KOJF+34ZlYMyG8Wss8jpp9oqoq2/cOvq+7f27T87Be3f1T3X2ku49m+3v7T7rb1aJ9rLv/NcnDVfVli13fmuSBDY7Eev1zkhdV1Wctfr5/a/xy5L631JPlWM7jPY56w2OxPi9J8qokf1NV713s++nFkxiB/eF1SX5zcXHjoSQ/vOF5WJPufk9VvT3Jvdn+VKC/iqfM7XueLAcAwEhujQAAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMNL/AaDoS9dPFLDoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "x, y = make_regression(n_samples=1000, n_features=10, n_informative=5, random_state=1)\n",
    "\n",
    "model = DecisionTreeRegressor()\n",
    "model.fit(x, y)\n",
    "\n",
    "importance = model.feature_importances_\n",
    "\n",
    "for i,v in enumerate(importance):\n",
    "    print('Feature: %0d, Score: %.5f' % (i,v))\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.bar([x for x in range(len(importance))], importance, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "a40edb20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature: 0, Score: 0.11842\n",
      "Feature: 1, Score: 0.06568\n",
      "Feature: 2, Score: 0.04537\n",
      "Feature: 3, Score: 0.08917\n",
      "Feature: 4, Score: 0.02627\n",
      "Feature: 5, Score: 0.32265\n",
      "Feature: 6, Score: 0.02220\n",
      "Feature: 7, Score: 0.01994\n",
      "Feature: 8, Score: 0.22675\n",
      "Feature: 9, Score: 0.06354\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHSCAYAAAAezFYoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVDElEQVR4nO3df6jd933f8dd70sxWLyGjVn/MsmZ3M83MiNYg3GwJLVmXzO7G1DHGFLoEsgZhiJtmzAxvf/Sf/jEKZmwFN8JkHpQ10x9ZDGJV45RuEEaaIrmNkziJi3CyWlOClTRrtnXUMX3vj3vMTt+57v0qvveeG93HAy73fL/fz+fcz+Eg6clX33O/1d0BAAD+vz+16QUAAMBBI5IBAGAQyQAAMIhkAAAYRDIAAAwiGQAAhqObXsB2brvttr7zzjs3vQwAAG5iTz311Fe7+9h2xw5kJN955525fPnyppcBAMBNrKr++ysdc7kFAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAw9FNLwCAvXXykTObXsKr9vRD5ze9BOCQcSYZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADIsiuaruq6pnq+pKVT28zfHTVfXpqvpUVV2uqrcsnQsAAAfNjpFcVUeSPJrk/iT3JHlHVd0zhv16kpPd/deS/JMkH7yBuQAAcKAsOZN8b5Ir3f1cd7+Y5HyS0+sDuvt/d3evNm9N0kvnAgDAQbMkkm9P8vza9tXVvj+mqv5+VX0hya9k62zy4rkAAHCQLInk2mZff8uO7ie6+/VJfiLJz93I3CSpqrOr65kvX79+fcGyAABgbyyJ5KtJ7ljbPp7k2isN7u6PJ/lLVXXbjczt7se6+1R3nzp27NiCZQEAwN5YEsmXktxdVXdV1S1JziS5sD6gqv5yVdXq8RuT3JLka0vmAgDAQXN0pwHd/VJVPZjkySRHkjze3c9U1QOr4+eS/IMk76qqbyb5v0n+0eqDfNvO3aPXAgAAu2LHSE6S7r6Y5OLYd27t8c8n+fmlcwEA4CBzxz0AABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYFgUyVV1X1U9W1VXqurhbY7/ZFV9evX1iao6uXbsS1X1mar6VFVd3s3FAwDAXji604CqOpLk0SRvS3I1yaWqutDdn1sb9sUkP9rdX6+q+5M8luSH146/tbu/uovrBgCAPbPkTPK9Sa5093Pd/WKS80lOrw/o7k9099dXm59Mcnx3lwkAAPtnSSTfnuT5te2rq32v5KeS/Oradif5WFU9VVVnb3yJAACwv3a83CJJbbOvtx1Y9dZsRfJb1na/ubuvVdX3JPm1qvpCd398m7lnk5xNkhMnTixYFgAA7I0lZ5KvJrljbft4kmtzUFW9IckHk5zu7q+9vL+7r62+v5DkiWxdvvEtuvux7j7V3aeOHTu2/BUAAMAuWxLJl5LcXVV3VdUtSc4kubA+oKpOJPlIknd29++s7b+1ql7z8uMkb0/y2d1aPAAA7IUdL7fo7peq6sEkTyY5kuTx7n6mqh5YHT+X5GeTfHeSX6yqJHmpu08l+d4kT6z2HU3yoe7+6J68EgAA2CVLrklOd19McnHsO7f2+D1J3rPNvOeSnJz7AQDgIHPHPQAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGI5uegEAAK/WyUfObHoJu+Lph85vegmsOJMMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMCyK5Kq6r6qeraorVfXwNsd/sqo+vfr6RFWdXDoXAAAOmh0juaqOJHk0yf1J7knyjqq6Zwz7YpIf7e43JPm5JI/dwFwAADhQlpxJvjfJle5+rrtfTHI+yen1Ad39ie7++mrzk0mOL50LAAAHzZJIvj3J82vbV1f7XslPJfnVb3MuAABs3NEFY2qbfb3twKq3ZiuS3/JtzD2b5GySnDhxYsGyAABgbyw5k3w1yR1r28eTXJuDquoNST6Y5HR3f+1G5iZJdz/W3ae6+9SxY8eWrB0AAPbEkki+lOTuqrqrqm5JcibJhfUBVXUiyUeSvLO7f+dG5gIAwEGz4+UW3f1SVT2Y5MkkR5I83t3PVNUDq+Pnkvxsku9O8otVlSQvrc4Kbzt3j14LAADsiiXXJKe7Lya5OPadW3v8niTvWToXAAAOMnfcAwCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAMOiSK6q+6rq2aq6UlUPb3P89VX1G1X1h1X10Dj2par6TFV9qqou79bCAQBgrxzdaUBVHUnyaJK3Jbma5FJVXejuz60N+70k70vyE6/wNG/t7q++yrUCAMC+WHIm+d4kV7r7ue5+Mcn5JKfXB3T3C919Kck392CNAACwr5ZE8u1Jnl/bvrrat1Qn+VhVPVVVZ19pUFWdrarLVXX5+vXrN/D0AACwu5ZEcm2zr2/gZ7y5u9+Y5P4k762qH9luUHc/1t2nuvvUsWPHbuDpAQBgdy2J5KtJ7ljbPp7k2tIf0N3XVt9fSPJEti7fAACAA2tJJF9KcndV3VVVtyQ5k+TCkievqlur6jUvP07y9iSf/XYXCwAA+2HH327R3S9V1YNJnkxyJMnj3f1MVT2wOn6uqr4vyeUkr03yR1X1/iT3JLktyRNV9fLP+lB3f3RPXgkAAOySHSM5Sbr7YpKLY9+5tcdfydZlGNM3kpx8NQsEAID95o57AAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADEc3vYCD5OQjZza9hF3x9EPnN70EAIDvaM4kAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMCyK5Kq6r6qeraorVfXwNsdfX1W/UVV/WFUP3chcAAA4aHaM5Ko6kuTRJPcnuSfJO6rqnjHs95K8L8kj38ZcAAA4UJacSb43yZXufq67X0xyPsnp9QHd/UJ3X0ryzRudCwAAB82SSL49yfNr21dX+5ZYPLeqzlbV5aq6fP369YVPDwAAu29JJNc2+3rh8y+e292Pdfep7j517NixhU8PAAC7b0kkX01yx9r28STXFj7/q5kLAAAbsSSSLyW5u6ruqqpbkpxJcmHh87+auQAAsBFHdxrQ3S9V1YNJnkxyJMnj3f1MVT2wOn6uqr4vyeUkr03yR1X1/iT3dPc3tpu7R68FAAB2xY6RnCTdfTHJxbHv3Nrjr2TrUopFcwEA4CBzxz0AABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwLLqZCHDzOPnImU0vYVc8/dD5TS8BgJuYM8kAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAw9FNL4DNO/nImU0vYVc8/dD5TS8BALhJOJMMAACDSAYAgMHlFgAA36FcMrl3nEkGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGBZFclXdV1XPVtWVqnp4m+NVVb+wOv7pqnrj2rEvVdVnqupTVXV5NxcPAAB74ehOA6rqSJJHk7wtydUkl6rqQnd/bm3Y/UnuXn39cJIPrL6/7K3d/dVdWzUAAOyhJWeS701ypbuf6+4Xk5xPcnqMOZ3kl3rLJ5O8rqq+f5fXCgAA+2JJJN+e5Pm17aurfUvHdJKPVdVTVXX2210oAADslx0vt0hS2+zrGxjz5u6+VlXfk+TXquoL3f3xb/khWwF9NklOnDixYFkAALA3lpxJvprkjrXt40muLR3T3S9/fyHJE9m6fONbdPdj3X2qu08dO3Zs2eoBAGAPLInkS0nurqq7quqWJGeSXBhjLiR51+q3XLwpye9395er6taqek2SVNWtSd6e5LO7uH4AANh1O15u0d0vVdWDSZ5MciTJ4939TFU9sDp+LsnFJD+e5EqSP0jy7tX0703yRFW9/LM+1N0f3fVXAQAAu2jJNcnp7ovZCuH1fefWHneS924z77kkJ1/lGgEAYF+54x4AAAyLziTDzejkI2c2vYRd8fRD5ze9BAC46TiTDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMRze9AID9cPKRM5tewq54+qHzm14CwKHgTDIAAAwiGQAABpEMAACDSAYAgMEH9wC4Kd0MH9b0QU3YHGeSAQBgEMkAADC43AIAbiI3w2UmiUtN2DxnkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAACGRZFcVfdV1bNVdaWqHt7meFXVL6yOf7qq3rh0LgAAHDQ7RnJVHUnyaJL7k9yT5B1Vdc8Ydn+Su1dfZ5N84AbmAgDAgbLkTPK9Sa5093Pd/WKS80lOjzGnk/xSb/lkktdV1fcvnAsAAAfKkki+Pcnza9tXV/uWjFkyFwAADpTq7j95QNU/TPK3u/s9q+13Jrm3u396bcyvJPlX3f3fVtu/nuSfJ/mBneauPcfZbF2qkSQ/mOTZV/naDqrbknx104tg33i/Dxfv9+HjPT9cvN83n7/Y3ce2O3B0weSrSe5Y2z6e5NrCMbcsmJsk6e7Hkjy2YD3f0arqcnef2vQ62B/e78PF+334eM8PF+/34bLkcotLSe6uqruq6pYkZ5JcGGMuJHnX6rdcvCnJ73f3lxfOBQCAA2XHM8nd/VJVPZjkySRHkjze3c9U1QOr4+eSXEzy40muJPmDJO/+k+buySsBAIBdsuRyi3T3xWyF8Pq+c2uPO8l7l8495G76S0r4Y7zfh4v3+/Dxnh8u3u9DZMcP7gEAwGHjttQAADCI5H3i9tyHS1XdUVX/tao+X1XPVNXPbHpN7L2qOlJVv11V/3nTa2FvVdXrqurDVfWF1Z/zv77pNbF3quqfrv4u/2xV/ceq+jObXhN7TyTvA7fnPpReSvLPuvuvJHlTkvd6zw+Fn0ny+U0vgn3xb5N8tLtfn+RkvO83raq6Pcn7kpzq7r+arV9EcGazq2I/iOT94fbch0x3f7m7f2v1+H9l6x9Qd5u8iVXV8SR/J8kHN70W9lZVvTbJjyT5d0nS3S929//c6KLYa0eT/NmqOprku/IK93zg5iKS94fbcx9iVXVnkh9K8psbXgp7699k606jf7ThdbD3fiDJ9ST/fnV5zQer6tZNL4q90d3/I8kjSX43yZezdS+Ij212VewHkbw/apt9fq3IIVBVfy7Jf0ry/u7+xqbXw96oqr+b5IXufmrTa2FfHE3yxiQf6O4fSvJ/kvisyU2qqv58tv73964kfyHJrVX1jze7KvaDSN4fS27tzU2mqv50tgL5l7v7I5teD3vqzUn+XlV9KVuXU/3NqvoPm10Se+hqkqvd/fL/Dn04W9HMzelvJflid1/v7m8m+UiSv7HhNbEPRPL+cHvuQ6aqKlvXK36+u//1ptfD3uruf9Hdx7v7zmz9+f4v3e1M002qu7+S5Pmq+sHVrh9L8rkNLom99btJ3lRV37X6u/3H4oOah8KiO+7x6rg996H05iTvTPKZqvrUat+/XN2BEvjO99NJfnl14uO5JO/e8HrYI939m1X14SS/la3fXPTbcee9Q8Ed9wAAYHC5BQAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABj+H9H8hyBMZN8VAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "x, y = make_classification(n_samples=1000, n_features=10, n_informative=5, random_state=1)\n",
    "\n",
    "model = DecisionTreeClassifier()\n",
    "model.fit(x, y)\n",
    "\n",
    "importance = model.feature_importances_\n",
    "\n",
    "for i,v in enumerate(importance):\n",
    "    print('Feature: %0d, Score: %.5f' % (i,v))\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.bar([x for x in range(len(importance))], importance, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1f3962fc",
   "metadata": {},
   "source": [
    "## Random Forest Feature Importance:"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aec48fbf",
   "metadata": {},
   "source": [
    "### Random Forest Regression Feature Importance:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "d7039a86",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature: 0, Score: 0.00295\n",
      "Feature: 1, Score: 0.00555\n",
      "Feature: 2, Score: 0.00292\n",
      "Feature: 3, Score: 0.00288\n",
      "Feature: 4, Score: 0.52943\n",
      "Feature: 5, Score: 0.42144\n",
      "Feature: 6, Score: 0.02603\n",
      "Feature: 7, Score: 0.00294\n",
      "Feature: 8, Score: 0.00308\n",
      "Feature: 9, Score: 0.00277\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsIAAAHSCAYAAADmLK3fAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAATG0lEQVR4nO3df6jd913H8dfbxKKbiGID06ZbqxZnlZWNWDcVfw/bOeyGipk6cTpKxc4plln9Y//sDxkUUbEawqwiikG2KWHLrDAFFX+Q7Een7ayETte7Thanbk7FLvr2j5yN493d7ml7Tk96348HhJzv9/vh3Hf4kpsn33zv+VZ3BwAApvmsbQ8AAADbIIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJEOb+sLX3nllX3NNdds68sDADDEO97xjn/u7iO7928thK+55pqcO3duW18eAIAhquof99rv1ggAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGOnwtgcAWKcb7jq+7RHW4r47Tm17BIADzxVhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMtFIIV9VNVfVgVZ2vqjv3OP7NVfWRqnr34tdr1z8qAACsz+H9FlTVoSR3J3lhkp0kZ6vqdHc/sGvpn3X3izcwIwAArN0qV4RvTHK+ux/q7keTnEpyy2bHAgCAzVolhK9K8vDS9s5i324vqKr7quptVfVVa5kOAAA2ZN9bI5LUHvt61/Y7kzyruz9WVS9K8gdJrvuUN6q6NcmtSfLMZz7zsU0KAABrtMoV4Z0kVy9tH03yyPKC7v5od39s8fpMks+uqit3v1F3n+zuY9197MiRI09gbAAAeGJWCeGzSa6rqmur6ookx5OcXl5QVc+oqlq8vnHxvh9e97AAALAu+94a0d0Xq+r2JPcmOZTknu6+v6puWxw/keR7kvxYVV1M8l9Jjnf37tsnAADgsrHKPcKfuN3hzK59J5Ze/0qSX1nvaAAAsDmeLAcAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYKTD2x4AgCfuhruOb3uEtbjvjlPbHgEYxBVhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADDSSiFcVTdV1YNVdb6q7vwM676mqv6nqr5nfSMCAMD67RvCVXUoyd1Jbk5yfZKXVdX1n2bd65Pcu+4hAQBg3Va5InxjkvPd/VB3P5rkVJJb9lj3qiRvSvKhNc4HAAAbsUoIX5Xk4aXtncW+T6qqq5K8NMmJ9Y0GAACbs0oI1x77etf2Lyb5me7+n8/4RlW3VtW5qjp34cKFFUcEAID1O7zCmp0kVy9tH03yyK41x5KcqqokuTLJi6rqYnf/wfKi7j6Z5GSSHDt2bHdMAwDAk2aVED6b5LqqujbJB5IcT/L9ywu6+9pPvK6q30zylt0RDAAAl5N9Q7i7L1bV7bn0aRCHktzT3fdX1W2L4+4LBgDgKWeVK8Lp7jNJzuzat2cAd/cPP/GxAABgszxZDgCAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGGmlEK6qm6rqwao6X1V37nH8lqp6T1W9u6rOVdU3rH9UAABYn8P7LaiqQ0nuTvLCJDtJzlbV6e5+YGnZ25Oc7u6uquck+b0kz97EwAAAsA6rXBG+Mcn57n6oux9NcirJLcsLuvtj3d2Lzacn6QAAwGVslRC+KsnDS9s7i33/T1W9tKr+Lslbk/zIesYDAIDNWCWEa499n3LFt7t/v7ufneQlSV635xtV3bq4h/jchQsXHtOgAACwTquE8E6Sq5e2jyZ55NMt7u4/TfJlVXXlHsdOdvex7j525MiRxzwsAACsyyohfDbJdVV1bVVdkeR4ktPLC6rqy6uqFq+fl+SKJB9e97AAALAu+35qRHdfrKrbk9yb5FCSe7r7/qq6bXH8RJLvTvJDVfXxJP+V5PuWfngOAAAuO/uGcJJ095kkZ3btO7H0+vVJXr/e0QAAYHM8WQ4AgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMNJKIVxVN1XVg1V1vqru3OP4D1TVexa//qKqblj/qAAAsD77hnBVHUpyd5Kbk1yf5GVVdf2uZe9L8k3d/Zwkr0tyct2DAgDAOq1yRfjGJOe7+6HufjTJqSS3LC/o7r/o7n9dbP5VkqPrHRMAANZrlRC+KsnDS9s7i32fzo8medteB6rq1qo6V1XnLly4sPqUAACwZquEcO2xr/dcWPUtuRTCP7PX8e4+2d3HuvvYkSNHVp8SAADW7PAKa3aSXL20fTTJI7sXVdVzkrwhyc3d/eH1jAcAAJuxyhXhs0muq6prq+qKJMeTnF5eUFXPTPLmJC/v7r9f/5gAALBe+14R7u6LVXV7knuTHEpyT3ffX1W3LY6fSPLaJF+U5FerKkkudvexzY0NAABPzCq3RqS7zyQ5s2vfiaXXr0zyyvWOBgAAm+PJcgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEgrhXBV3VRVD1bV+aq6c4/jz66qv6yq/66qO9Y/JgAArNfh/RZU1aEkdyd5YZKdJGer6nR3P7C07F+S/ESSl2xiSAAAWLdVrgjfmOR8dz/U3Y8mOZXkluUF3f2h7j6b5OMbmBEAANZulRC+KsnDS9s7i30AAPCUtUoI1x77+vF8saq6tarOVdW5CxcuPJ63AACAtVglhHeSXL20fTTJI4/ni3X3ye4+1t3Hjhw58njeAgAA1mKVED6b5LqquraqrkhyPMnpzY4FAACbte+nRnT3xaq6Pcm9SQ4luae776+q2xbHT1TVM5KcS/L5Sf63qn4yyfXd/dHNjQ4AAI/fviGcJN19JsmZXftOLL3+p1y6ZQIAAJ4SPFkOAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMJIQBgBgJCEMAMBIQhgAgJGEMAAAIwlhAABGEsIAAIwkhAEAGEkIAwAwkhAGAGAkIQwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMdHjbAwDA43XDXce3PcJa3HfHqW2PACO5IgwAwEhCGACAkYQwAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICRhDAAACMJYQAARhLCAACMJIQBABhJCAMAMNLhbQ/Ak+eGu45ve4Qn7L47Tm17BADggFjpinBV3VRVD1bV+aq6c4/jVVW/vDj+nqp63vpHBQCA9dn3inBVHUpyd5IXJtlJcraqTnf3A0vLbk5y3eLX1yb5tcXvlyVXRgHgqcm/4azTKrdG3JjkfHc/lCRVdSrJLUmWQ/iWJL/V3Z3kr6rqC6rqi7v7g2ufGB6jg/BNM3ns3zin/rlhAn+/5zkI5/xyPN+r3BpxVZKHl7Z3Fvse6xoAALhs1KWLuJ9hQdX3JvmO7n7lYvvlSW7s7lctrXlrkp/v7j9fbL89yWu6+x273uvWJLcuNr8iyYPr+oNcZq5M8s/bHoInlXM+i/M9i/M9i/N9MD2ru4/s3rnKrRE7Sa5e2j6a5JHHsSbdfTLJyRW+5lNaVZ3r7mPbnoMnj3M+i/M9i/M9i/M9yyq3RpxNcl1VXVtVVyQ5nuT0rjWnk/zQ4tMjnp/kI+4PBgDgcrbvFeHuvlhVtye5N8mhJPd09/1Vddvi+IkkZ5K8KMn5JP+Z5BWbGxkAAJ64lR6o0d1ncil2l/edWHrdSX58vaM9pR342z/4FM75LM73LM73LM73IPv+sBwAABxEKz1ZDgAADhohvGb7PY6ag6Oqrq6qP6mq91bV/VX16m3PxOZV1aGqeldVvWXbs7B5iwdEvbGq/m7xd/0F256Jzamqn1p8P//bqvrdqvqcbc/EZgnhNVp6HPXNSa5P8rKqun67U7FBF5P8dHd/ZZLnJ/lx53uEVyd577aH4EnzS0n+sLufneSGOPcHVlVdleQnkhzr7q/OpQ8IeOo/zo3PSAiv1ycfR93djyb5xOOoOYC6+4Pd/c7F63/PpX8gPVHxAKuqo0m+M8kbtj0Lm1dVn5/kG5P8epJ096Pd/W9bHYpNO5zkc6vqcJKnZY9nInCwCOH18qjpoarqmiTPTfLXWx6FzfrFJK9J8r9bnoMnx5cmuZDkNxa3w7yhqp6+7aHYjO7+QJK7krw/yQdz6ZkIf7Tdqdg0Ibxetcc+H8txwFXV5yV5U5Kf7O6PbnseNqOqXpzkQ7sfHc+BdjjJ85L8Wnc/N8l/JPGzHwdUVX1hLv0v7rVJviTJ06vqB7c7FZsmhNdrpUdNc3BU1WfnUgT/Tne/edvzsFFfn+S7quofcum2p2+tqt/e7khs2E6Sne7+xP/0vDGXwpiD6duTvK+7L3T3x5O8OcnXbXkmNkwIr9cqj6PmgKiqyqV7B9/b3b+w7XnYrO7+2e4+2t3X5NLf7T/ubleLDrDu/qckD1fVVyx2fVuSB7Y4Epv1/iTPr6qnLb6/f1v8cOSBt9KT5VjNp3sc9ZbHYnO+PsnLk/xNVb17se/nFk9iBA6GVyX5ncXFjYeSvGLL87Ah3f3XVfXGJO/MpU8Felc8Ze7A82Q5AABGcmsEAAAjCWEAAEYSwgAAjCSEAQAYSQgDADCSEAYAYCQhDADASEIYAICR/g/9+0piI+yIjwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=10, n_informative=5, random_state=1)\n",
    "\n",
    "model = RandomForestRegressor()\n",
    "model.fit(x, y)\n",
    "\n",
    "importance = model.feature_importances_\n",
    "\n",
    "for i,v in enumerate(importance):\n",
    "    print('Feature: %0d, Score: %.5f' % (i,v))\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.bar([x for x in range(len(importance))], importance, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0769d9c8",
   "metadata": {},
   "source": [
    "### Random Forest Classification Feature Importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "78eed373",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature: 0, Score: 0.07355\n",
      "Feature: 1, Score: 0.12478\n",
      "Feature: 2, Score: 0.11943\n",
      "Feature: 3, Score: 0.08798\n",
      "Feature: 4, Score: 0.03125\n",
      "Feature: 5, Score: 0.24125\n",
      "Feature: 6, Score: 0.03636\n",
      "Feature: 7, Score: 0.02749\n",
      "Feature: 8, Score: 0.12954\n",
      "Feature: 9, Score: 0.12836\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHSCAYAAAAezFYoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAATvElEQVR4nO3df6hf933f8dd70sxWNyGlVn/MsmcXTDMzIhqEm86lxe2S2d2Y+s9AoUshNAhD3DRjZvX2R//pH/vHjK3gRojUg7Jm+iOLQWxqnLINykhTJLdxEidxEU5Wa0qw3GbNWKGOyLt/3K/Zd+9cV0fRvfd7o/t4gNA953zO974vXyw/OTr6nuruAAAA/89f2/QAAACw34hkAAAYRDIAAAwiGQAABpEMAACDSAYAgOHwpgfYzh133NH33HPPpscAAOAW9txzz73a3Ue2O7YvI/mee+7JxYsXNz0GAAC3sKr6n290bNHtFlX1cFW9WFWXquqJbY7/XFV9ZvXrk1V1bO3Yl6vqs1X16apSvgAA7HvXvZJcVYeSPJXknUkuJ7lQVee6+/Nry76U5Ce7+2tV9UiSM0l+dO34Q9396g7ODQAAu2bJleQHklzq7pe6+7UkZ5OcWF/Q3Z/s7q+tNj+V5OjOjgkAAHtnSSTfmeTlte3Lq31v5BeS/Pbadif5RFU9V1WnbnxEAADYW0v+4V5ts6+3XVj1ULYi+cfXdj/Y3Veq6vuS/E5VfbG7f3ebc08lOZUkd99994KxAABgdyy5knw5yV1r20eTXJmLquptST6c5ER3/8nr+7v7yur3V5I8k63bN75Fd5/p7uPdffzIkW0/iQMAAPbEkki+kOS+qrq3qm5LcjLJufUFVXV3ko8leU93/9Ha/tur6k2vf53kXUk+t1PDAwDAbrju7Rbdfa2qHkvybJJDSZ7u7heq6tHV8dNJfiXJ9yb59apKkmvdfTzJ9yd5ZrXvcJKPdPfHd+UnAQCAHVLd295evFHHjx9vDxMBAGA3VdVzqwu732LRw0QAAOAgEckAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADAc3vQAAOyuY0+e3PQIN+35x89uegTggHElGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAADDokiuqoer6sWqulRVT2xz/Oeq6jOrX5+sqmNLzwUAgP3mupFcVYeSPJXkkST3J3l3Vd0/ln0pyU9299uS/GqSMzdwLgAA7CtLriQ/kORSd7/U3a8lOZvkxPqC7v5kd39ttfmpJEeXngsAAPvNkki+M8nLa9uXV/veyC8k+e0bPbeqTlXVxaq6ePXq1QVjAQDA7lgSybXNvt52YdVD2YrkX77Rc7v7THcf7+7jR44cWTAWAADsjsML1lxOctfa9tEkV+aiqnpbkg8neaS7/+RGzgUAgP1kyZXkC0nuq6p7q+q2JCeTnFtfUFV3J/lYkvd09x/dyLkAALDfXPdKcndfq6rHkjyb5FCSp7v7hap6dHX8dJJfSfK9SX69qpLk2urWiW3P3aWfBQAAdsSS2y3S3eeTnB/7Tq99/b4k71t6LgAA7GeeuAcAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwLIrkqnq4ql6sqktV9cQ2x99aVb9XVX9RVY+PY1+uqs9W1aer6uJODQ4AALvl8PUWVNWhJE8leWeSy0kuVNW57v782rI/TfKBJD/7Bi/zUHe/epOzAgDAnlhyJfmBJJe6+6Xufi3J2SQn1hd09yvdfSHJN3ZhRgAA2FNLIvnOJC+vbV9e7Vuqk3yiqp6rqlNvtKiqTlXVxaq6ePXq1Rt4eQAA2FlLIrm22dc38D0e7O63J3kkyfur6ie2W9TdZ7r7eHcfP3LkyA28PAAA7KwlkXw5yV1r20eTXFn6Dbr7yur3V5I8k63bNwAAYN9aEskXktxXVfdW1W1JTiY5t+TFq+r2qnrT618neVeSz327wwIAwF647qdbdPe1qnosybNJDiV5urtfqKpHV8dPV9UPJLmY5M1JvllVH0xyf5I7kjxTVa9/r49098d35ScBAIAdct1ITpLuPp/k/Nh3eu3rr2brNozp60mO3cyAAACw1zxxDwAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGA4vOkBAABu1rEnT256hB3x/ONnNz0CK64kAwDA4EoyAMB3KFfQd48ryQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgMFHwOHjYwAABleSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwOBhIhxYHqICALwRV5IBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAyLIrmqHq6qF6vqUlU9sc3xt1bV71XVX1TV4zdyLgAA7DfXjeSqOpTkqSSPJLk/ybur6v6x7E+TfCDJk9/GuQAAsK8suZL8QJJL3f1Sd7+W5GySE+sLuvuV7r6Q5Bs3ei4AAOw3SyL5ziQvr21fXu1b4mbOBQCAjVgSybXNvl74+ovPrapTVXWxqi5evXp14csDAMDOWxLJl5PctbZ9NMmVha+/+NzuPtPdx7v7+JEjRxa+PAAA7LwlkXwhyX1VdW9V3ZbkZJJzC1//Zs4FAICNOHy9Bd19raoeS/JskkNJnu7uF6rq0dXx01X1A0kuJnlzkm9W1QeT3N/dX9/u3F36WQAAYEdcN5KTpLvPJzk/9p1e+/qr2bqVYtG5AACwn3niHgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABgOb3oAYG8de/LkpkfYEc8/fnbTIwBwC3MlGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAyHNz3AfnLsyZObHmFHPP/42U2PAADwHc2VZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAIZFkVxVD1fVi1V1qaqe2OZ4VdWvrY5/pqrevnbsy1X12ar6dFVd3MnhAQBgNxy+3oKqOpTkqSTvTHI5yYWqOtfdn19b9kiS+1a/fjTJh1a/v+6h7n51x6YGAIBdtORK8gNJLnX3S939WpKzSU6MNSeS/GZv+VSSt1TVD+7wrAAAsCeWRPKdSV5e27682rd0TSf5RFU9V1Wnvt1BAQBgr1z3dosktc2+voE1D3b3lar6viS/U1Vf7O7f/ZZvshXQp5Lk7rvvXjAWAADsjiVXki8nuWtt+2iSK0vXdPfrv7+S5Jls3b7xLbr7THcf7+7jR44cWTY9AADsgiWRfCHJfVV1b1XdluRkknNjzbkkP7/6lIt3JPmz7v5KVd1eVW9Kkqq6Pcm7knxuB+cHAIAdd93bLbr7WlU9luTZJIeSPN3dL1TVo6vjp5OcT/IzSS4l+fMk712d/v1Jnqmq17/XR7r74zv+UwAAwA5ack9yuvt8tkJ4fd/pta87yfu3Oe+lJMduckYAANhTnrgHAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgOHwpgcAgN1w7MmTmx7hpj3/+NlNjwAHlivJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYPAwEeBAuBUeLJF4uATAXnElGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMPgIOAC4hfi4Q9gZriQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYFgUyVX1cFW9WFWXquqJbY5XVf3a6vhnqurtS88FAID95rqRXFWHkjyV5JEk9yd5d1XdP5Y9kuS+1a9TST50A+cCAMC+suRK8gNJLnX3S939WpKzSU6MNSeS/GZv+VSSt1TVDy48FwAA9pUlkXxnkpfXti+v9i1Zs+RcAADYV6q7/+oFVf8kyT/o7vettt+T5IHu/sW1Nf8lyb/u7v+x2v6vSf5Fkh+63rlrr3EqW7dqJMkPJ3nxJn+2/eqOJK9uegj2jPf7YPF+Hzze84PF+33r+dvdfWS7A4cXnHw5yV1r20eTXFm45rYF5yZJuvtMkjML5vmOVlUXu/v4pudgb3i/Dxbv98HjPT9YvN8Hy5LbLS4kua+q7q2q25KcTHJurDmX5OdXn3LxjiR/1t1fWXguAADsK9e9ktzd16rqsSTPJjmU5OnufqGqHl0dP53kfJKfSXIpyZ8nee9fde6u/CQAALBDltxuke4+n60QXt93eu3rTvL+pececLf8LSX8f7zfB4v3++Dxnh8s3u8D5Lr/cA8AAA4aj6UGAIBBJO8Rj+c+WKrqrqr671X1hap6oap+adMzsfuq6lBV/WFV/edNz8Luqqq3VNVHq+qLq//Of2zTM7F7quqfrf4s/1xV/ceq+hubnondJ5L3gMdzH0jXkvzz7v47Sd6R5P3e8wPhl5J8YdNDsCf+XZKPd/dbkxyL9/2WVVV3JvlAkuPd/Xez9UEEJzc7FXtBJO8Nj+c+YLr7K939B6uv/0+2/gfqaZO3sKo6muQfJvnwpmdhd1XVm5P8RJLfSJLufq27//dGh2K3HU7yN6vqcJLvyhs884Fbi0jeGx7PfYBV1T1JfiTJ7294FHbXv83Wk0a/ueE52H0/lORqkn+/ur3mw1V1+6aHYnd09/9K8mSSP07ylWw9C+ITm52KvSCS90Zts8/HihwAVfXdSf5Tkg9299c3PQ+7o6r+UZJXuvu5Tc/Cnjic5O1JPtTdP5Lk/ybxb01uUVX1Pdn62997k/ytJLdX1T/d7FTsBZG8N5Y82ptbTFX99WwF8m9198c2PQ+76sEk/7iqvpyt26l+qqr+w2ZHYhddTnK5u1//26GPZiuauTX9/SRf6u6r3f2NJB9L8vc2PBN7QCTvDY/nPmCqqrJ1v+IXuvvfbHoedld3/8vuPtrd92Trv+//1t2uNN2iuvurSV6uqh9e7frpJJ/f4Ejsrj9O8o6q+q7Vn+0/Hf9Q80BY9MQ9bo7Hcx9IDyZ5T5LPVtWnV/v+1eoJlMB3vl9M8lurCx8vJXnvhudhl3T371fVR5P8QbY+uegP48l7B4In7gEAwOB2CwAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADD8JX4fAQmyZ1S3AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "x, y = make_classification(n_samples=1000, n_features=10, n_informative=5, random_state=1)\n",
    "\n",
    "model = RandomForestClassifier()\n",
    "model.fit(x, y)\n",
    "\n",
    "importance = model.feature_importances_\n",
    "\n",
    "for i,v in enumerate(importance):\n",
    "    print('Feature: %0d, Score: %.5f' % (i,v))\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.bar([x for x in range(len(importance))], importance, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "238cbe34",
   "metadata": {},
   "source": [
    "## Permutation Feature Importance:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "a4b5ec9a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature: 0, Score: 176.01688\n",
      "Feature: 1, Score: 343.23450\n",
      "Feature: 2, Score: 174.13697\n",
      "Feature: 3, Score: 52.37386\n",
      "Feature: 4, Score: 9813.95806\n",
      "Feature: 5, Score: 7982.98863\n",
      "Feature: 6, Score: 967.36082\n",
      "Feature: 7, Score: 107.07873\n",
      "Feature: 8, Score: 110.71381\n",
      "Feature: 9, Score: 117.72698\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtIAAAHSCAYAAADBgiw3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAWBklEQVR4nO3df6jl9X3n8dd7nTY1CW7NZiJ2RnYsO6Q1gqQZxDZQllrQpaXmjwqzkESKZSC4aVoaivaf/CVkQbptYBUkaR23IbODDURK062YlmVBTK/5gVErGWJXp07jdHebSv8wa/reP+7X5Xa8mvi+c+dc73084HC+53O+33M/l4Pjk+/9nPOt7g4AAPDG/ItVTwAAAN6MhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwMC+VU9g6p3vfGcfOnRo1dMAAGCXe+yxx/6uu/efO/6mDelDhw5lbW1t1dMAAGCXq6r/udm4pR0AADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMfN+Qrqrfr6oXquobG8beUVUPVdU3l/tLNzx3R1Wdqqqnq+qGDePvq6rHl+c+VVW1jL+lqv7rMv5oVR06z78jAACcdz/IGen7ktx4ztjtSR7u7sNJHl4ep6quSnI0yXuWY+6uqouWY+5JcizJ4eX2ymvemuT/dPe/SfKfkvzH6S8DAAAXyvcN6e7+70n+9znDNyU5vmwfT/KBDeMnuvul7n4myakk11bV5Uku6e5HuruT3H/OMa+81gNJrn/lbDUAAOxU0zXSl3X3mSRZ7t+1jB9I8tyG/U4vYweW7XPH/9kx3f1yku8k+VfDeQEAwAVxvj9suNmZ5H6d8dc75tUvXnWsqtaqau3s2bPDKQIAwNZNQ/rby3KNLPcvLOOnk1yxYb+DSZ5fxg9uMv7PjqmqfUn+ZV69lCRJ0t33dveR7j6yf//+4dQBAGDrpiH9YJJblu1bknxhw/jR5Zs4rsz6hwq/vCz/eLGqrlvWP3/4nGNeea1fTvKlZR01AADsWPu+3w5V9bkk/zbJO6vqdJJPJPlkkpNVdWuSZ5PcnCTd/URVnUzyZJKXk9zW3d9bXuojWf8GkIuTfHG5JclnkvyXqjqV9TPRR8/LbwYAANuo3qwnf48cOdJra2urngYAALtcVT3W3UfOHXdlQwAAGBDSAAAwIKQBAGDg+37YEGAvueau3fF5569//MSqpwCw6zkjDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMDAlkK6qn6jqp6oqm9U1eeq6keq6h1V9VBVfXO5v3TD/ndU1amqerqqbtgw/r6qenx57lNVVVuZFwAAbLdxSFfVgSS/luRId1+d5KIkR5PcnuTh7j6c5OHlcarqquX59yS5McndVXXR8nL3JDmW5PByu3E6LwAAuBC2urRjX5KLq2pfkrcmeT7JTUmOL88fT/KBZfumJCe6+6XufibJqSTXVtXlSS7p7ke6u5Pcv+EYAADYkcYh3d1/k+SuJM8mOZPkO939Z0ku6+4zyz5nkrxrOeRAkuc2vMTpZezAsn3uOAAA7FhbWdpxadbPMl+Z5MeSvK2qPvh6h2wy1q8zvtnPPFZVa1W1dvbs2Tc6ZQAAOG/2beHYn0/yTHefTZKq+nySn0ny7aq6vLvPLMs2Xlj2P53kig3HH8z6UpDTy/a546/S3fcmuTdJjhw5smlsA/DGXXPX0VVP4bz4+sdPrHoKwB6ylTXSzya5rqreunzLxvVJnkryYJJbln1uSfKFZfvBJEer6i1VdWXWP1T45WX5x4tVdd3yOh/ecAwAAOxI4zPS3f1oVT2Q5CtJXk7y1ayfLX57kpNVdWvWY/vmZf8nqupkkieX/W/r7u8tL/eRJPcluTjJF5cbAADsWFtZ2pHu/kSST5wz/FLWz05vtv+dSe7cZHwtydVbmQsAAFxIrmwIAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAY2FJIV9WPVtUDVfVXVfVUVf10Vb2jqh6qqm8u95du2P+OqjpVVU9X1Q0bxt9XVY8vz32qqmor8wIAgO221TPSv5fkT7v7J5Jck+SpJLcnebi7Dyd5eHmcqroqydEk70lyY5K7q+qi5XXuSXIsyeHlduMW5wUAANtqHNJVdUmSn03ymSTp7u92998nuSnJ8WW340k+sGzflOREd7/U3c8kOZXk2qq6PMkl3f1Id3eS+zccAwAAO9JWzkj/eJKzSf6gqr5aVZ+uqrcluay7zyTJcv+uZf8DSZ7bcPzpZezAsn3uOAAA7FhbCel9SX4qyT3d/d4k/5hlGcdr2Gzdc7/O+KtfoOpYVa1V1drZs2ff6HwBAOC82UpIn05yursfXR4/kPWw/vayXCPL/Qsb9r9iw/EHkzy/jB/cZPxVuvve7j7S3Uf279+/hakDAMDWjEO6u/82yXNV9e5l6PokTyZ5MMkty9gtSb6wbD+Y5GhVvaWqrsz6hwq/vCz/eLGqrlu+rePDG44BAIAdad8Wj/9oks9W1Q8n+VaSX8l6nJ+sqluTPJvk5iTp7ieq6mTWY/vlJLd19/eW1/lIkvuSXJzki8sNAAB2rC2FdHd/LcmRTZ66/jX2vzPJnZuMryW5eitzAQCAC8mVDQEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAICBLYd0VV1UVV+tqj9eHr+jqh6qqm8u95du2PeOqjpVVU9X1Q0bxt9XVY8vz32qqmqr8wIAgO10Ps5IfyzJUxse357k4e4+nOTh5XGq6qokR5O8J8mNSe6uqouWY+5JcizJ4eV243mYFwAAbJsthXRVHUzyC0k+vWH4piTHl+3jST6wYfxEd7/U3c8kOZXk2qq6PMkl3f1Id3eS+zccAwAAO9JWz0j/bpLfSvJPG8Yu6+4zSbLcv2sZP5DkuQ37nV7GDizb544DAMCONQ7pqvrFJC9092M/6CGbjPXrjG/2M49V1VpVrZ09e/YH/LEAAHD+beWM9PuT/FJV/XWSE0l+rqr+MMm3l+UaWe5fWPY/neSKDccfTPL8Mn5wk/FX6e57u/tIdx/Zv3//FqYOAABbMw7p7r6juw9296Gsf4jwS939wSQPJrll2e2WJF9Yth9McrSq3lJVV2b9Q4VfXpZ/vFhV1y3f1vHhDccAAMCOtG8bXvOTSU5W1a1Jnk1yc5J09xNVdTLJk0leTnJbd39vOeYjSe5LcnGSLy43AADYsc5LSHf3XyT5i2X7fyW5/jX2uzPJnZuMryW5+nzMBQAALgRXNgQAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAyMQ7qqrqiqP6+qp6rqiar62DL+jqp6qKq+udxfuuGYO6rqVFU9XVU3bBh/X1U9vjz3qaqqrf1aAACwvbZyRvrlJL/Z3T+Z5Lokt1XVVUluT/Jwdx9O8vDyOMtzR5O8J8mNSe6uqouW17onybEkh5fbjVuYFwAAbLtxSHf3me7+yrL9YpKnkhxIclOS48tux5N8YNm+KcmJ7n6pu59JcirJtVV1eZJLuvuR7u4k9284BgAAdqTzska6qg4leW+SR5Nc1t1nkvXYTvKuZbcDSZ7bcNjpZezAsn3u+GY/51hVrVXV2tmzZ8/H1AEAYGTLIV1Vb0/yR0l+vbv/4fV23WSsX2f81YPd93b3ke4+sn///jc+WQAAOE+2FNJV9UNZj+jPdvfnl+FvL8s1sty/sIyfTnLFhsMPJnl+GT+4yTgAAOxYW/nWjkrymSRPdffvbHjqwSS3LNu3JPnChvGjVfWWqroy6x8q/PKy/OPFqrpuec0PbzgGAAB2pH1bOPb9ST6U5PGq+toy9ttJPpnkZFXdmuTZJDcnSXc/UVUnkzyZ9W/8uK27v7cc95Ek9yW5OMkXlxsAAOxY45Du7v+Rzdc3J8n1r3HMnUnu3GR8LcnV07kAAMCF5sqGAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAgJAGAIABIQ0AAANCGgAABvategIAsCrX3HV01VM4L77+8ROrngLsSc5IAwDAgJAGAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAwIaQAAGBDSAAAwIKQBAGBASAMAwICQBgCAASENAAADQhoAAAaENAAADAhpAAAYENIAADAgpAEAYEBIAwDAwL5VT4A3j2vuOrrqKWzZ1z9+YtVTAAB2CWekAQBgQEgDAMCAkAYAgAEhDQAAA0IaAAAGhDQAAAz4+rsBXwO3t3i/AYDNOCMNAAADQhoAAAYs7QCAPWY3LFlL3viytb36eye743fficsUd8wZ6aq6saqerqpTVXX7qucDAACvZ0eEdFVdlOQ/J/l3Sa5K8u+r6qrVzgoAAF7bTlnacW2SU939rSSpqhNJbkry5EpnBXvYbvgzYLIz/xQIwO6wI85IJzmQ5LkNj08vYwAAsCNVd696Dqmqm5Pc0N2/ujz+UJJru/uj5+x3LMmx5eG7kzx9QSd64bwzyd+tehJcUN7zvcX7vbd4v/cW7/fu9K+7e/+5gztlacfpJFdseHwwyfPn7tTd9ya590JNalWqaq27j6x6Hlw43vO9xfu9t3i/9xbv996yU5Z2/GWSw1V1ZVX9cJKjSR5c8ZwAAOA17Ygz0t39clX9hyT/LclFSX6/u59Y8bQAAOA17YiQTpLu/pMkf7LqeewQu375Cq/iPd9bvN97i/d7b/F+7yE74sOGAADwZrNT1kgDAMCbipDeYVwqfe+oqiuq6s+r6qmqeqKqPrbqObH9quqiqvpqVf3xqufC9quqH62qB6rqr5b/1n961XNi+1TVbyz/nn+jqj5XVT+y6jmxvYT0DuJS6XvOy0l+s7t/Msl1SW7zfu8JH0vy1KonwQXze0n+tLt/Isk18d7vWlV1IMmvJTnS3Vdn/csTdsclYnlNQnpn+f+XSu/u7yZ55VLp7ELdfaa7v7Jsv5j1/8G6oucuVlUHk/xCkk+vei5sv6q6JMnPJvlMknT3d7v771c6KbbbviQXV9W+JG/NJtfEYHcR0juLS6XvUVV1KMl7kzy64qmwvX43yW8l+acVz4ML48eTnE3yB8tynk9X1dtWPSm2R3f/TZK7kjyb5EyS73T3n612Vmw3Ib2z1CZjvlZll6uqtyf5oyS/3t3/sOr5sD2q6heTvNDdj616Llww+5L8VJJ7uvu9Sf4xic++7FJVdWnW/4p8ZZIfS/K2qvrgamfFdhPSO8sPdKl0do+q+qGsR/Rnu/vzq54P2+r9SX6pqv4668u2fq6q/nC1U2KbnU5yurtf+UvTA1kPa3ann0/yTHef7e7/m+TzSX5mxXNimwnpncWl0veQqqqsr518qrt/Z9XzYXt19x3dfbC7D2X9v+0vdbezVbtYd/9tkueq6t3L0PVJnlzhlNhezya5rqreuvz7fn18uHTX2zFXNsSl0veg9yf5UJLHq+pry9hvL1f5BHaHjyb57HJy5FtJfmXF82GbdPejVfVAkq9k/VuZvhpXOdz1XNkQAAAGLO0AAIABIQ0AAANCGgAABoQ0AAAMCGkAABgQ0gAAMCCkAQBgQEgDAMDA/wOaSx4srmAeBQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_regression\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.inspection import permutation_importance\n",
    "\n",
    "x, y = make_regression(n_samples=1000, n_features=10, n_informative=5, random_state=1)\n",
    "\n",
    "model = KNeighborsRegressor()\n",
    "\n",
    "model.fit(x, y)\n",
    "\n",
    "results = permutation_importance(model, x, y, scoring='neg_mean_squared_error')\n",
    "importance = results.importances_mean\n",
    "for i,v in enumerate(importance):\n",
    "    print('Feature: %0d, Score: %.5f' % (i,v))\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.bar([x for x in range(len(importance))], importance, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c10c41f",
   "metadata": {},
   "source": [
    "### Permutation Feature Importance for Regression:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "04b91395",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature: 0, Score: 0.05260\n",
      "Feature: 1, Score: 0.06480\n",
      "Feature: 2, Score: 0.05260\n",
      "Feature: 3, Score: 0.08900\n",
      "Feature: 4, Score: 0.05420\n",
      "Feature: 5, Score: 0.06100\n",
      "Feature: 6, Score: 0.07760\n",
      "Feature: 7, Score: 0.05600\n",
      "Feature: 8, Score: 0.05520\n",
      "Feature: 9, Score: 0.02540\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAskAAAHSCAYAAAAezFYoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAATJklEQVR4nO3dT6il913H8c/XGYNtRSJ0FjETnBSG6iCEhiFMLbiwBTNRnJUwgTYQkCGQtFUMEt24dBNEAyFDaFMIFmcRuxh0MIJ/Fi4SMmmaaBwDQ6zNmBHHhVGaRRr8urhHvH5z4z1pzrlnOvf1ggv3PM/vmfs9PEzy5pnnnqe6OwAAwP/6oU0PAAAA1xuRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADAc3PQAO/n4xz/eR44c2fQYAADcwF588cV/6+5DO+27LiP5yJEjuXjx4qbHAADgBlZV//R++9xuAQAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwHBw0wMAe+uOR09veoSVePnhc5seAYAbmCvJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAMNSkVxVd1fVa1V1uaoe2WF/VdVji/2vVNWd2/b9elW9WlV/V1V/VFU/sso3AAAAq7ZrJFfVgSSPJzmZ5FiSe6vq2Fh2MsnRxdeZJE8sjr01yZeSHO/un0lyIMnplU0PAABrsMyV5LuSXO7u17v7nSTnkpwaa04lebq3PJfk5qq6ZbHvYJKPVNXBJB9N8uaKZgcAgLVYJpJvTfLGttdXFtt2XdPd/5zk0STfSXI1yVvd/eff/7gAALB+y0Ry7bCtl1lTVT+eravMtyf5iSQfq6rP7/hDqs5U1cWqunjt2rUlxgIAgPVYJpKvJLlt2+vDee8tE++35nNJ/rG7r3X395J8I8nP7vRDuvvJ7j7e3ccPHTq07PwAALByy0TyC0mOVtXtVXVTtn7x7vxYcz7JfYtPuTiRrdsqrmbrNosTVfXRqqokn01yaYXzAwDAyh3cbUF3v1tVDyV5NlufTvFUd79aVQ8s9p9NciHJPUkuJ3k7yf2Lfc9X1TNJvpnk3SQvJXlyHW8EAABWZddITpLuvpCtEN6+7ey27zvJg+9z7O8k+Z0PMSMAAOwpT9wDAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMS30EHAD8oLnj0dObHuFDe/nhc5seAfYtV5IBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMBzc9ABs3h2Pnt70CCvx8sPnNj0CAHCDcCUZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAAAGkQwAAINIBgCAQSQDAMAgkgEAYBDJAAAwiGQAABhEMgAADCIZAACGg5seAID1uuPR05se4UN7+eFzmx4B2GeWupJcVXdX1WtVdbmqHtlhf1XVY4v9r1TVndv23VxVz1TVP1TVpar69CrfAAAArNqukVxVB5I8nuRkkmNJ7q2qY2PZySRHF19nkjyxbd8fJPmz7v6pJHckubSCuQEAYG2WuZJ8V5LL3f16d7+T5FySU2PNqSRP95bnktxcVbdU1Y8l+bkkX02S7n6nu/99deMDAMDqLRPJtyZ5Y9vrK4tty6z5RJJrSb5WVS9V1Veq6mMfYl4AAFi7ZSK5dtjWS645mOTOJE9096eSfDfJe+5pTpKqOlNVF6vq4rVr15YYCwAA1mOZSL6S5LZtrw8neXPJNVeSXOnu5xfbn8lWNL9Hdz/Z3ce7+/ihQ4eWmR0AANZimUh+IcnRqrq9qm5KcjrJ+bHmfJL7Fp9ycSLJW919tbv/JckbVfXJxbrPJvn7VQ0PAADrsOvnJHf3u1X1UJJnkxxI8lR3v1pVDyz2n01yIck9SS4neTvJ/dv+iC8m+foisF8f+wAA4Lqz1MNEuvtCtkJ4+7az277vJA++z7HfSnL8+x8RAAD2lsdSAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwHNz0AALA6dzx6etMjrMTLD5/b9Ajsc64kAwDAIJIBAGAQyQAAMLgnGQD4gedebFbNlWQAABhEMgAADCIZAAAGkQwAAINf3AP2Bb/UA8AH4UoyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBg8BFw2/iIqP3F+QYA3o8ryQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgGGpSK6qu6vqtaq6XFWP7LC/quqxxf5XqurOsf9AVb1UVX+yqsEBAGBddo3kqjqQ5PEkJ5McS3JvVR0by04mObr4OpPkibH/y0kufehpAQBgDyxzJfmuJJe7+/XufifJuSSnxppTSZ7uLc8lubmqbkmSqjqc5BeTfGWFcwMAwNosE8m3Jnlj2+sri23Lrvn9JL+Z5L++vxEBAGBvLRPJtcO2XmZNVf1Skn/t7hd3/SFVZ6rqYlVdvHbt2hJjAQDAeiwTyVeS3Lbt9eEkby655jNJfrmqvp2t2zR+vqr+cKcf0t1Pdvfx7j5+6NChJccHAIDVWyaSX0hytKpur6qbkpxOcn6sOZ/kvsWnXJxI8lZ3X+3u3+ruw919ZHHcX3b351f5BgAAYNUO7ragu9+tqoeSPJvkQJKnuvvVqnpgsf9skgtJ7klyOcnbSe5f38gAALBeu0ZyknT3hWyF8PZtZ7d930ke3OXP+Oskf/2BJwQAgD3miXsAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwHBw0wMAAPD9uePR05seYSVefvjcpkd4D1eSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAAaRDAAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMS0VyVd1dVa9V1eWqemSH/VVVjy32v1JVdy6231ZVf1VVl6rq1ar68qrfAAAArNqukVxVB5I8nuRkkmNJ7q2qY2PZySRHF19nkjyx2P5ukt/o7p9OciLJgzscCwAA15VlriTfleRyd7/e3e8kOZfk1FhzKsnTveW5JDdX1S3dfbW7v5kk3f2fSS4luXWF8wMAwMotE8m3Jnlj2+sreW/o7rqmqo4k+VSS5z/wlAAAsIeWieTaYVt/kDVV9aNJ/jjJr3X3f+z4Q6rOVNXFqrp47dq1JcYCAID1WCaSryS5bdvrw0neXHZNVf1wtgL56939jff7Id39ZHcf7+7jhw4dWmZ2AABYi2Ui+YUkR6vq9qq6KcnpJOfHmvNJ7lt8ysWJJG9199WqqiRfTXKpu39vpZMDAMCaHNxtQXe/W1UPJXk2yYEkT3X3q1X1wGL/2SQXktyT5HKSt5Pcvzj8M0m+kORvq+pbi22/3d0XVvouAABghXaN5CRZRO2Fse3stu87yYM7HPc32fl+ZQAAuG554h4AAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgEEkAwDAIJIBAGAQyQAAMIhkAAAYRDIAAAwiGQAABpEMAACDSAYAgGGpSK6qu6vqtaq6XFWP7LC/quqxxf5XqurOZY8FAIDrza6RXFUHkjye5GSSY0nurapjY9nJJEcXX2eSPPEBjgUAgOvKMleS70pyubtf7+53kpxLcmqsOZXk6d7yXJKbq+qWJY8FAIDryjKRfGuSN7a9vrLYtsyaZY4FAIDrSnX3/7+g6leS/EJ3/+ri9ReS3NXdX9y25k+T/G53/83i9V8k+c0kn9jt2G1/xpls3aqRJJ9M8tqHfG/Xq48n+bdND8Gecb73F+d7/3HO9xfn+8bzk919aKcdB5c4+EqS27a9PpzkzSXX3LTEsUmS7n4yyZNLzPMDraoudvfxTc/B3nC+9xfne/9xzvcX53t/WeZ2ixeSHK2q26vqpiSnk5wfa84nuW/xKRcnkrzV3VeXPBYAAK4ru15J7u53q+qhJM8mOZDkqe5+taoeWOw/m+RCknuSXE7ydpL7/79j1/JOAABgRZa53SLdfSFbIbx929lt33eSB5c9dp+74W8p4f9wvvcX53v/cc73F+d7H9n1F/cAAGC/8VhqAAAYRPIe8Xju/aWqbquqv6qqS1X1alV9edMzsX5VdaCqXqqqP9n0LKxXVd1cVc9U1T8s/p5/etMzsT5V9euL/5b/XVX9UVX9yKZnYv1E8h7weO596d0kv9HdP53kRJIHnfN94ctJLm16CPbEHyT5s+7+qSR3xHm/YVXVrUm+lOR4d/9Mtj6I4PRmp2IviOS94fHc+0x3X+3uby6+/89s/Q/U0yZvYFV1OMkvJvnKpmdhvarqx5L8XJKvJkl3v9Pd/77RoVi3g0k+UlUHk3w07/PMB24sInlveDz3PlZVR5J8KsnzGx6F9fr9bD1p9L82PAfr94kk15J8bXF7zVeq6mObHor16O5/TvJoku8kuZqtZ0H8+WanYi+I5L1RO2zzsSL7QFX9aJI/TvJr3f0fm56H9aiqX0ryr9394qZnYU8cTHJnkie6+1NJvpvE75rcoKrqx7P1r7+3J/mJJB+rqs9vdir2gkjeG8s82psbTFX9cLYC+evd/Y1Nz8NafSbJL1fVt7N1O9XPV9UfbnYk1uhKkivd/T//OvRMtqKZG9Pnkvxjd1/r7u8l+UaSn93wTOwBkbw3PJ57n6mqytb9ipe6+/c2PQ/r1d2/1d2Hu/tItv5+/2V3u9J0g+ruf0nyRlV9crHps0n+foMjsV7fSXKiqj66+G/7Z+MXNfeFpZ64x4fj8dz70meSfCHJ31bVtxbbfnvxBErgB98Xk3x9ceHj9ST3b3ge1qS7n6+qZ5J8M1ufXPRSPHlvX/DEPQAAGNxuAQAAg0gGAIBBJAMAwCCSAQBgEMkAADCIZAAAGEQyAAAMIhkAAIb/Bt3yolBqQjx1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.inspection import permutation_importance\n",
    "\n",
    "x, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, random_state=1)\n",
    "\n",
    "model = KNeighborsClassifier()\n",
    "\n",
    "model.fit(x, y)\n",
    "\n",
    "results = permutation_importance(model, x, y, scoring='accuracy')\n",
    "\n",
    "importance = results.importances_mean\n",
    "for i,v in enumerate(importance):\n",
    "    print('Feature: %0d, Score: %.5f' % (i,v))\n",
    "\n",
    "plt.figure(figsize=(12,8))\n",
    "plt.bar([x for x in range(len(importance))], importance, color=\"seagreen\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d015db9c",
   "metadata": {},
   "source": [
    "### Permutation Feature Importance for Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "9b0592bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 84.55\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "x, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, random_state=1)\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "model = LogisticRegression(solver='liblinear')\n",
    "model.fit(xtrain, ytrain)\n",
    "\n",
    "yhat = model.predict(xtest)\n",
    "\n",
    "accuracy = accuracy_score(ytest, yhat)\n",
    "print('Accuracy: %.2f' % (accuracy*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0d9aea6f",
   "metadata": {},
   "source": [
    "## Feature Selection with Importance:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "05808cc7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 84.55\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "def select_features(xtrain, ytrain, xtest):\n",
    "    fs = SelectFromModel(RandomForestClassifier(n_estimators=1000), max_features=5)\n",
    "    fs.fit(xtrain, ytrain)\n",
    "    xtrain_fs = fs.transform(xtrain)\n",
    "    xtest_fs = fs.transform(xtest)\n",
    "    return xtrain_fs, xtest_fs, fs\n",
    "\n",
    "\n",
    "x, y = make_classification(n_samples=1000, n_features=10, n_informative=5, n_redundant=5, random_state=1)\n",
    "\n",
    "xtrain, xtest, ytrain, ytest = train_test_split(x, y, test_size=0.33, random_state=1)\n",
    "\n",
    "xtrain_fs, xtest_fs, fs = select_features(xtrain, ytrain, xtest)\n",
    "\n",
    "model = LogisticRegression(solver='liblinear')\n",
    "model.fit(xtrain_fs, ytrain)\n",
    "\n",
    "yhat = model.predict(xtest_fs)\n",
    "\n",
    "accuracy = accuracy_score(ytest, yhat)\n",
    "print('Accuracy: %.2f' % (accuracy*100))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}