Humana_RandomForest.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "skhGOkPf9o9E"
      },
      "source": [
        "# **Humana Competition**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qUPeHRHL3UaN"
      },
      "source": [
        "## Feature Importance Ranking\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kny07On1PTt9"
      },
      "source": [
        "Example: random forest feature importance\n",
        "https://mljar.com/blog/feature-importance-in-random-forest/\n",
        "\n",
        "\n",
        "Example: xgboost feature importance \n",
        "https://machinelearningmastery.com/feature-importance-and-feature-selection-with-xgboost-in-python/\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "id": "E_tFZl0S5niY"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.inspection import permutation_importance\n",
        "from matplotlib import pyplot as plt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T-_pPW3Ll267"
      },
      "source": [
        "### Split Data into training and test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {
        "id": "ofB2QEU86ZxD"
      },
      "outputs": [],
      "source": [
        "X = cleaned_training.drop(columns=['id', 'hi_flag']) # all features other than id and hi_flag\n",
        "y = cleaned_training[\"hi_flag\"] # hi_flag\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=12)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ThkcUOAg95pW",
        "outputId": "9a7c5a0c-0a4c-427e-f4c8-3d965f89935b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "Int64Index: 36225 entries, 8854 to 14155\n",
            "Columns: 681 entries, rx_hum_19_pmpm_cost to cms_race_cd_6\n",
            "dtypes: float64(612), int64(37), uint8(32)\n",
            "memory usage: 180.7 MB\n"
          ]
        }
      ],
      "source": [
        "X_train.info()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9vQrJwLHP5wI"
      },
      "source": [
        "### Note: Deal with Imbalances"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aos9DyZ8QVX0",
        "outputId": "bb629a1c-4dd2-4768-9776-d5e66b790410"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[34643, 1582]"
            ]
          },
          "execution_count": 53,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "[sum(y_train.values == 0), sum(y_train.values != 0)] "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PBbnOFdWP9d4",
        "outputId": "800f9e6a-aace-469e-e950-a6a5d3f89c25"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[11539, 536]"
            ]
          },
          "execution_count": 54,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "[sum(y_test.values == 0), sum(y_test.values != 0)] \n",
        "# the proportion of 1 and 0 are similar in the test and train set"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YnsJSVZP5tfZ"
      },
      "source": [
        "### Random Forest"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 55,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CgfawROGYvBl",
        "outputId": "0c01c23a-a4c8-4575-a64f-46b2c721cd08"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.6842965549667384"
            ]
          },
          "execution_count": 55,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from imblearn.ensemble import BalancedRandomForestClassifier\n",
        "\n",
        "BRFC = BalancedRandomForestClassifier(n_estimators = 150, random_state = 2, class_weight = {0: 34643, 1:1582})\n",
        "BRFC.fit(X_train, y_train)\n",
        "\n",
        "y_pred = BRFC.predict_proba(X_test)[:, 1] \n",
        "\n",
        "roc_auc_score(y_test, y_pred)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X3_JvZgX-LVk"
      },
      "outputs": [],
      "source": [
        "# what are less important features to avoid overfitting? \n",
        "less_importance = X_train.columns[BRFC.feature_importances_ == 0]\n",
        "importances = BRFC.feature_importances_\n",
        "\n",
        "sorted_indices = np.argsort(importances)[::-1]\n",
        "\n",
        "# print(*X_train.columns[sorted_indices], sep = \"\\n\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mD83Cm-wX4au",
        "outputId": "325964bb-a0a4-43aa-ebc3-0a994a48c8e2"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "500 most important features: [330, 97, 271, 164, 433, 485, 183, 586, 644, 357, 289, 286, 439, 519, 309, 620, 431, 647, 316, 536, 560, 297, 12, 386, 561, 472, 170, 648, 20, 356, 402, 199, 614, 9, 75, 491, 145, 306, 451, 618, 368, 287, 265, 305, 310, 121, 315, 216, 200, 630, 273, 354, 360, 326, 161, 281, 13, 551, 118, 86, 14, 621, 82, 596, 505, 516, 162, 191, 106, 21, 34, 260, 404, 220, 209, 292, 352, 339, 219, 450, 269, 425, 126, 249, 434, 50, 120, 405, 304, 221, 129, 500, 604, 313, 568, 550, 320, 240, 638, 85, 19, 369, 119, 373, 457, 194, 51, 45, 130, 480, 285, 482, 166, 426, 184, 460, 376, 455, 168, 197, 635, 160, 595, 71, 28, 136, 531, 26, 329, 43, 603, 153, 81, 63, 530, 251, 308, 3, 573, 8, 481, 294, 580, 83, 232, 489, 585, 637, 323, 242, 396, 139, 606, 365, 375, 643, 528, 151, 575, 245, 157, 223, 332, 142, 22, 23, 272, 348, 203, 350, 198, 201, 27, 148, 80, 334, 88, 300, 241, 338, 529, 390, 333, 178, 179, 616, 372, 533, 437, 30, 512, 243, 165, 341, 353, 527, 282, 206, 303, 583, 393, 467, 96, 37, 253, 255, 628, 544, 359, 204, 59, 479, 588, 317, 349, 447, 389, 601, 584, 413, 602, 52, 599, 646, 6, 237, 261, 351, 258, 364, 92, 324, 293, 244, 68, 298, 264, 409, 458, 429, 77, 403, 196, 144, 391, 337, 380, 566, 234, 167, 327, 56, 388, 641, 215, 383, 572, 459, 212, 180, 74, 427, 202, 639, 57, 492, 525, 558, 38, 679, 539, 415, 627, 190, 111, 95, 382, 592, 608, 524, 498, 578, 299, 1, 456, 307, 116, 581, 446, 361, 150, 279, 424, 636, 549, 538, 438, 504, 565, 229, 416, 488, 246, 193, 381, 181, 399, 432, 185, 462, 645, 238, 506, 210, 542, 331, 463, 385, 48, 252, 674, 394, 177, 478, 266, 275, 476, 131, 274, 122, 417, 154, 301, 406, 115, 11, 655, 521, 486, 2, 445, 46, 366, 104, 513, 562, 172, 64, 40, 665, 571, 98, 132, 659, 526, 312, 421, 72, 564, 76, 146, 653, 453, 319, 408, 189, 123, 503, 124, 15, 207, 414, 410, 17, 624, 344, 103, 291, 335, 487, 398, 420, 574, 36, 501, 518, 340, 468, 195, 471, 545, 515, 33, 664, 466, 78, 66, 559, 18, 448, 87, 99, 673, 79, 44, 156, 629, 230, 90, 192, 374, 358, 322, 205, 135, 231, 535, 613, 400, 543, 73, 147, 412, 428, 342, 612, 213, 4, 355, 649, 55, 555, 678, 484, 507, 395, 140, 387, 444, 392, 69, 250, 280, 143, 100, 651, 314, 553, 345, 632, 125, 54, 138, 256, 378, 631, 502, 642, 47, 302, 464, 672, 175, 152, 661, 654, 587, 475, 671, 336, 537, 483, 367, 236, 248, 188, 182, 493, 105, 343, 268, 660, 227, 546, 541, 509, 497, 680, 384, 669, 556, 570, 41, 477, 158, 276, 263, 465, 662, 675]\n"
          ]
        }
      ],
      "source": [
        "important_features_dict = {}\n",
        "for idx, val in enumerate(BRFC.feature_importances_):\n",
        "    important_features_dict[idx] = val\n",
        "\n",
        "important_features_list = sorted(important_features_dict,\n",
        "                                 key=important_features_dict.get,\n",
        "                                 reverse=True)\n",
        "important1 = important_features_list[:500]\n",
        "print(f'500 most important features: {important_features_list[:500]}')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ve57czOdYUOg",
        "outputId": "d9261b08-c34a-42ff-c27c-f29ed24aa526"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.6855582560376038"
            ]
          },
          "execution_count": 152,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#choose top n important feature\n",
        "X_train_new = X_train.iloc[:,important1]\n",
        "len(X_train_new)\n",
        "X_test_new = X_test.iloc[:,important1]\n",
        "BRFC2 = BalancedRandomForestClassifier(n_estimators = 1000, random_state = 42, class_weight = {0: 34643, 1:1582})\n",
        "BRFC2.fit(X_train_new, y_train)\n",
        "y_pred2 = BRFC2.predict_proba(X_test_new)[:, 1]\n",
        "roc_auc_score(y_test, y_pred2)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3.10.0 64-bit",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.7"
    },
    "vscode": {
      "interpreter": {
        "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}