notebooks/mdi_plus_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# MDI+: Example Usages"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n",
    "from sklearn.metrics import r2_score, mean_absolute_error, accuracy_score, roc_auc_score\n",
    "\n",
    "from imodels.importance import RandomForestPlusRegressor, RandomForestPlusClassifier, \\\n",
    "    RidgeRegressorPPM, LassoRegressorPPM, IdentityTransformer\n",
    "from imodels.importance.rf_plus import _fast_r2_score"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "def neg_mae(y_true, y_pred, **kwargs):\n",
    "    \"\"\"\n",
    "    Evaluates negative mean absolute error\n",
    "    \"\"\"\n",
    "    return -mean_absolute_error(y_true, y_pred, **kwargs)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "# helper variables\n",
    "rf_regressor = RandomForestRegressor(n_estimators=100, min_samples_leaf=5, max_features=0.33, random_state=331)\n",
    "rf_classifier = RandomForestClassifier(n_estimators=100, min_samples_leaf=1, max_features=\"sqrt\", random_state=331)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 1. Regression Example"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "# generate data from linear model: y = x1 + x2 + N(0, 1)\n",
    "n = 200\n",
    "p = 10\n",
    "s = 2\n",
    "X = np.random.normal(size=(n, p))\n",
    "beta = np.concatenate((np.ones(s), np.zeros(p-s)))\n",
    "y = np.matmul(X, beta) + np.random.normal(size=n)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=12345)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 1.1 MDI+ with default settings for regression"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "# fit RF+\n",
    "rf_plus_model = RandomForestPlusRegressor(rf_model=rf_regressor)\n",
    "rf_plus_model.fit(X_train, y_train)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "0.602898866954644"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make predictions with RF+\n",
    "preds = rf_plus_model.predict(X_test)\n",
    "r2_score(y_test, preds)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "data": {
      "text/plain": "   var  importance\n0    0    0.354405\n1    1    0.321326\n8    8    0.026648\n3    3    0.000100\n2    2   -0.002104\n9    9   -0.003035\n6    6   -0.003280\n5    5   -0.004150\n4    4   -0.005290\n7    7   -0.005371",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>var</th>\n      <th>importance</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0.354405</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>0.321326</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>8</td>\n      <td>0.026648</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>0.000100</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>-0.002104</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>9</td>\n      <td>-0.003035</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>6</td>\n      <td>-0.003280</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>5</td>\n      <td>-0.004150</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>-0.005290</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>7</td>\n      <td>-0.005371</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get MDI+ scores (higher r^2 value = greater importance)\n",
    "mdi_plus_scores = rf_plus_model.get_mdi_plus_scores(X_train, y_train)\n",
    "mdi_plus_scores.sort_values(\"importance\", ascending=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 1.2 MDI+ with custom partial prediction model and evaluation metric(s)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "# fit RF+ with custom partial prediction model\n",
    "rf_plus_model = RandomForestPlusRegressor(rf_model=rf_regressor, prediction_model=LassoRegressorPPM())\n",
    "rf_plus_model.fit(X_train, y_train)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "data": {
      "text/plain": "   var  r2_score  negative_mae\n0    0  0.358490     -1.140341\n1    1  0.328313     -1.159828\n8    8  0.021085     -1.423189\n3    3 -0.001271     -1.445924\n2    2 -0.002602     -1.447519\n9    9 -0.002837     -1.446741\n5    5 -0.003045     -1.447727\n6    6 -0.003163     -1.447022\n7    7 -0.003550     -1.447024\n4    4 -0.003911     -1.447655",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>var</th>\n      <th>r2_score</th>\n      <th>negative_mae</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0.358490</td>\n      <td>-1.140341</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>0.328313</td>\n      <td>-1.159828</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>8</td>\n      <td>0.021085</td>\n      <td>-1.423189</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>-0.001271</td>\n      <td>-1.445924</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>-0.002602</td>\n      <td>-1.447519</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>9</td>\n      <td>-0.002837</td>\n      <td>-1.446741</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>5</td>\n      <td>-0.003045</td>\n      <td>-1.447727</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>6</td>\n      <td>-0.003163</td>\n      <td>-1.447022</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>7</td>\n      <td>-0.003550</td>\n      <td>-1.447024</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>-0.003911</td>\n      <td>-1.447655</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get MDI+ scores with custom evaluation metrics/scorers\n",
    "mdi_plus_scores = rf_plus_model.get_mdi_plus_scores(X_train, y_train, scoring_fns={\"r2_score\": _fast_r2_score, \"negative_mae\": neg_mae})\n",
    "mdi_plus_scores.sort_values(\"r2_score\", ascending=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 1.3 MDI+ with custom transformer"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "The example below is equivalent to running RF+ with `include_raw=True`"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [],
   "source": [
    "# fit RF+ with custom transformer\n",
    "rf_plus_model = RandomForestPlusRegressor(rf_model=rf_regressor, include_raw=False, add_transformers=[IdentityTransformer()])\n",
    "rf_plus_model.fit(X_train, y_train)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "   var  importance\n0    0    0.355724\n1    1    0.325702\n8    8    0.025157\n3    3    0.000388\n2    2   -0.002160\n9    9   -0.003352\n6    6   -0.003477\n5    5   -0.004867\n7    7   -0.005865\n4    4   -0.005900",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>var</th>\n      <th>importance</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0.355724</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>0.325702</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>8</td>\n      <td>0.025157</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>0.000388</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>-0.002160</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>9</td>\n      <td>-0.003352</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>6</td>\n      <td>-0.003477</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>5</td>\n      <td>-0.004867</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>7</td>\n      <td>-0.005865</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>-0.005900</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get MDI+ scores\n",
    "mdi_plus_scores = rf_plus_model.get_mdi_plus_scores(X_train, y_train)\n",
    "mdi_plus_scores.sort_values(\"importance\", ascending=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 1.4 Choosing the GLM and scoring metric via stability score"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "There are many choices of GLMs and scoring metrics that can be made within the MDI+ framework.\n",
    "\n",
    "One way to select the GLM and scoring metric in MDI+ is by evaluating the stability of the feature importances/rankings for each choice of GLM/metric and taking the GLM/metric that is the most stable across different bootstrap samples of trees. For example, we can take the GLM and metric with the highest stability score, as measured by RBO below."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [],
   "source": [
    "n_bootstraps = 25\n",
    "prediction_models = {\"ridge\": RidgeRegressorPPM(), \"lasso\": LassoRegressorPPM()}\n",
    "scoring_fns = {\"r2\": _fast_r2_score, \"neg_mae\": neg_mae}\n",
    "stability_dict = {}\n",
    "for model_name, prediction_model in prediction_models.items():\n",
    "    # fit RF+\n",
    "    rf_plus_model = RandomForestPlusRegressor(rf_model=rf_regressor, prediction_model=prediction_model)\n",
    "    rf_plus_model.fit(X_train, y_train)\n",
    "    # get MDI+ scores\n",
    "    mdi_plus_scores = rf_plus_model.get_mdi_plus_scores(X_train, y_train, scoring_fns=scoring_fns)\n",
    "    # get MDI+ stability scores\n",
    "    mdi_plus_stability_scores = rf_plus_model.get_mdi_plus_stability_scores(B=n_bootstraps)\n",
    "    stability_dict[model_name] = mdi_plus_stability_scores"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "     ppm   scorer       RBO     tauAP\n0  ridge       r2  0.943672  0.901981\n2  lasso       r2  0.902463  0.811391\n3  lasso  neg_mae  0.900443  0.784878\n1  ridge  neg_mae  0.899285  0.851119",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>ppm</th>\n      <th>scorer</th>\n      <th>RBO</th>\n      <th>tauAP</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>ridge</td>\n      <td>r2</td>\n      <td>0.943672</td>\n      <td>0.901981</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>lasso</td>\n      <td>r2</td>\n      <td>0.902463</td>\n      <td>0.811391</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>lasso</td>\n      <td>neg_mae</td>\n      <td>0.900443</td>\n      <td>0.784878</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>ridge</td>\n      <td>neg_mae</td>\n      <td>0.899285</td>\n      <td>0.851119</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat(stability_dict, axis=0).reset_index().rename(columns={\"level_0\": \"ppm\"}).drop(columns=[\"level_1\"]).sort_values(\"RBO\", ascending=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 1.5 Aggregating multiple MDI+ rankings in an ensemble"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Instead of choosing a single GLM and metric to use in MDI+, it may be preferable in some cases to aggregate MDI+ feature importances/rankings across multiple choices of GLMs and metrics.\n",
    "\n",
    "One naive method for doing this ensembling is to take the median rank across each choice of GLM and metric (as shown below). However, more creative ensembling schemes can also be explored."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [],
   "source": [
    "prediction_models = {\"ridge\": RidgeRegressorPPM(), \"lasso\": LassoRegressorPPM()}\n",
    "scoring_fns = {\"r2\": _fast_r2_score, \"neg_mae\": neg_mae}\n",
    "mdi_plus_scores_dict = {}\n",
    "for model_name, prediction_model in prediction_models.items():\n",
    "    # fit RF+\n",
    "    rf_plus_model = RandomForestPlusRegressor(rf_model=rf_regressor, prediction_model=prediction_model)\n",
    "    rf_plus_model.fit(X_train, y_train)\n",
    "    # get MDI+ scores\n",
    "    mdi_plus_scores = rf_plus_model.get_mdi_plus_scores(X_train, y_train, scoring_fns=scoring_fns)\n",
    "    for col in mdi_plus_scores.columns:\n",
    "        if col != \"var\":\n",
    "            mdi_plus_scores = mdi_plus_scores.rename(columns={col: model_name + \"_\" + col})\n",
    "    mdi_plus_scores_dict[model_name] = mdi_plus_scores"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "data": {
      "text/plain": "   var  median_rank\n0    0          1.0\n1    1          2.0\n8    8          3.0\n3    3          4.0\n2    2          6.0\n9    9          6.0\n6    6          6.5\n7    7          8.5\n4    4          9.0\n5    5          9.0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>var</th>\n      <th>median_rank</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>1.0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>2.0</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>8</td>\n      <td>3.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>4.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>6.0</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>9</td>\n      <td>6.0</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>6</td>\n      <td>6.5</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>7</td>\n      <td>8.5</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>9.0</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>5</td>\n      <td>9.0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mdi_plus_scores_df = pd.concat([df.set_index('var') for df in mdi_plus_scores_dict.values()], axis=1)\n",
    "mdi_plus_ranks_df = mdi_plus_scores_df.rank(ascending=False).median(axis=1)\n",
    "mdi_plus_ranks_df = pd.DataFrame(mdi_plus_ranks_df, columns=[\"median_rank\"]).reset_index()\n",
    "mdi_plus_ranks_df.sort_values(\"median_rank\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## 2. Classification Example"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [],
   "source": [
    "# generate data from logistic model: logit(E[Y|X]) = x1 + x2\n",
    "n = 200\n",
    "p = 10\n",
    "s = 2\n",
    "X = np.random.normal(size=(n, p))\n",
    "beta = np.concatenate((np.ones(s), np.zeros(p-s)))\n",
    "probs = 1 / (1 + np.exp(-np.matmul(X, beta)))\n",
    "y = (np.random.uniform(size=n) < probs) * 1\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=12345)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 2.1 MDI+ with default classification settings"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "# fit RF+\n",
    "rf_plus_model = RandomForestPlusClassifier(rf_model=rf_classifier)\n",
    "rf_plus_model.fit(X_train, y_train)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [
    {
     "data": {
      "text/plain": "(0.7878787878787878, 0.8585858585858586)"
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make predictions with RF+\n",
    "preds = rf_plus_model.predict(X_test)\n",
    "prob_preds = rf_plus_model.predict_proba(X_test)\n",
    "accuracy_score(y_test, preds), roc_auc_score(y_test, prob_preds[:, 1])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "data": {
      "text/plain": "   var  importance\n0    0   -0.643608\n1    1   -0.664461\n4    4   -0.681521\n9    9   -0.693593\n3    3   -0.695175\n6    6   -0.695259\n7    7   -0.698201\n2    2   -0.698987\n8    8   -0.700452\n5    5   -0.701411",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>var</th>\n      <th>importance</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>-0.643608</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>-0.664461</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>-0.681521</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>9</td>\n      <td>-0.693593</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>-0.695175</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>6</td>\n      <td>-0.695259</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>7</td>\n      <td>-0.698201</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>-0.698987</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>8</td>\n      <td>-0.700452</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>5</td>\n      <td>-0.701411</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get MDI+ scores (higher ngative log-loss value = greater importance)\n",
    "mdi_plus_scores = rf_plus_model.get_mdi_plus_scores(X_train, y_train)\n",
    "mdi_plus_scores.sort_values(\"importance\", ascending=False)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}