diff --git a/howto/03-howto-gridsearch.ipynb b/howto/03-howto-gridsearch.ipynb new file mode 100644 index 00000000..b028bdc8 --- /dev/null +++ b/howto/03-howto-gridsearch.ipynb @@ -0,0 +1,1409 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# About\n", + "\n", + "This notebook demonstrates several additional tools to optimize classification model provided by __Reproducible experiment platform (REP)__ package:\n", + "\n", + "* __grid search for the best classifier hyperparameters__\n", + "\n", + "* __different optimization algorithms__ \n", + "\n", + "* __different scoring models__ (optimization of arbirtary figure of merit)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Populating the interactive namespace from numpy and matplotlib\n" + ] + } + ], + "source": [ + "%pylab inline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Loading data " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "File `magic04.data' already there; not retrieving.\r\n" + ] + } + ], + "source": [ + "!cd toy_datasets; wget -O magic04.data -nc https://archive.ics.uci.edu/ml/machine-learning-databases/magic/magic04.data" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "['fLength',\n", + " 'fWidth',\n", + " 'fSize',\n", + " 'fConc',\n", + " 'fConc1',\n", + " 'fAsym',\n", + " 'fM3Long',\n", + " 'fM3Trans',\n", + " 'fAlpha',\n", + " 'fDist']" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import numpy, pandas\n", + "from rep.utils import train_test_split\n", + "from sklearn.metrics import roc_auc_score\n", + "\n", + "columns = ['fLength', 'fWidth', 'fSize', 'fConc', 'fConc1', 'fAsym', 'fM3Long', 'fM3Trans', 'fAlpha', 'fDist', 'g']\n", + "data = pandas.read_csv('toy_datasets/magic04.data', names=columns)\n", + "labels = numpy.array(data['g'] == 'g', dtype=int)\n", + "data = data.drop('g', axis=1)\n", + "\n", + "train_data, test_data, train_labels, test_labels = train_test_split(data, labels)\n", + "\n", + "list(data.columns)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Variables used in training" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "features = list(set(columns) - {'g'})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Metric definition\n", + "\n", + "In Higgs challenge the aim is to maximize `AMS` metrics.
\n", + "To measure the quality one should choose not only classifier, but also an optimal threshold, \n", + "where the maximal value of `AMS` is achieved.\n", + "\n", + "Such metrics (which require a threshold) are called threshold-based.\n", + "\n", + "__rep.utils__ contain class __OptimalMetric__, which computes the maximal value for threshold-based metric (and may be used as metric).\n", + "\n", + "Use this class to generate metric and use it in grid search." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Prepare quality metric\n", + "\n", + "first we define AMS metric, and `utils.OptimalMetric` generates " + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from rep.report import metrics" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "def AMS(s, b, s_norm=sum(test_labels == 1), b_norm=6*sum(test_labels == 0)): \n", + " return s * s_norm / numpy.sqrt(b * b_norm + 10.)\n", + "optimal_AMS = metrics.OptimalMetric(AMS)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(3071, 1684)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sum(test_labels == 1),sum(test_labels == 0)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Compute threshold vs metric quality\n", + "random predictions for signal and background were used here" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Optimal cut=0.0096, quality=30.5758\n" + ] + }, + { + "data": { + "text/html": [], + "text/plain": [ + "" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": [ + "iVBORw0KGgoAAAANSUhEUgAAAY0AAAEZCAYAAABrUHmEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n", + "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm4HFWd//F3JyEsYVd2kBARkEXCoiggHHCU1YURFFxY\n", + "FBGXUcZR+KGDZBDFDRlRcEQNUcYRcRdQBCVBFh0RRBHBDYKAbIqBAGG9398fp+503Ur1vb3W93TX\n", + "5/U8/dyupbs+Vbk5555azgEREREREREREREREREREREREREREREREREZNu8Afg08mL2uAfbPLV8A\n", + "jBVe11QbUUREJjOjwm3dARwP/BGYBhwJfBd4PrEyMeAy4I25zzxRYT4REUnc34G3ZO8XABf6RRER\n", + "kalMc9rudOBQYCXgp9k8A3YH7gV+D5wDrOOSTkREkrAd8DDwJPAQcEBu2WuBA4Ftsp83ADcCMyvO\n", + "KCIiiVgBmAPsAHwEWArs3GLdDYjXNA6qJpqIiEylygvhEFsYt2bvf0W8CP4O4KiSde8G7gQ2b/Fd\n", + "dwEb9jugiMiI+zOty9XkXQ58ucWydYDHgTe0WG4DSTSc5nkHSMg87wAJmecdICHzvAMkpKeys8qW\n", + "xkeBi4ith9WA1wF7AvsCs4D/AL4J3APMBk4jXhT/ToUZh9Vs7wAJme0dICGzvQMkZLZ3gFFRZaWx\n", + "HvDfwPrEh/t+TawwLiPeRbUt8RmNNYmnpi4HDgYeqTCjiIiMKJ2eagreARISvAMkJHgHSEjwDpCQ\n", + "2padtd1xEZEe9FR2ej3cJ/0VvAMkJHgHSEjwDlChB4iFoV7N1wM9HdEWqr7lVkRkENYCGpMsD8Ci\n", + "SpKkQ2djCnRARGScyoPltTomOj0lIiLVUKUxGoJ3gIQE7wAJCd4BEhK8A4wKVRoiIlILOocpIuOG\n", + "pTxYRLyrKd979wLiSKWvKKx7Rjb/iGx6JnA6cUC7pcBt2Tqt6JqGiMgQmw28ALiPiRWEAX8ADs/N\n", + "mwG8BvgTzUL+RGBHYkevqxFPuV03yMBlVGmMhuAdICHBO0BCgneAhATvAMRK4cfAeTRbD+MuJA5C\n", + "t2Y2vS+xq6V7c+vsTBwi+55s+nZi10yVUqUhIiPPYKGB9frqMcbhwNeBC4B9mDgy6WPA94gjmo6v\n", + "+5XC538OvAd4G3FAu8meS5ESw3IOU0QGb9LyoB8VRo+Vxu7AMuJpJYgjkx6XvT8X+BCwG3ANsAax\n", + "NbEScCXN01bTgLcDVxErmbuYeEqrZLc7mt8WtTREZOQ1oNGPVw8RjgAuJV7ABvgGzVNUDWJBfjWx\n", + "9fHvxNNVjxW+Yww4m1gBrQF8GJgPbNVDrlpRS6MpeAdISPAOkJDgHaBCU5UHoYoQLaxMHA5iKXHY\n", + "h7uJd1A9DTyP2NI4JVv35Gz+i7PpfEujzP20HhJ7IC0N9T0lIjJYrwKeArYHnsjmNYjXNg7PTQOc\n", + "CfyUWFkUHUccJvsXxKGzXw+sms2TNqilISLjUi4Pfgh8omT+IcRWx1dotjSK8i2NtwC/BJYA/yBe\n", + "GN9/ku0OpKUxzGq74yKyHJUHy9OFcGkpeAdISPAOkJDgHSAhwTvAqFClISIitaDmqIiMU3mwPJ2e\n", + "EhERX1VWGu8g9qXyYPa6huWv/M8jPuX4KLAQ2HqqLzVY2WDdviYdPsE7QEKCd4CEBO8ACQneAUZF\n", + "lZXGHcDxwA7ATsDlxM63ts+Wn0DsV+WdxF4c7wMuI96HXCp7rP9R4G6DewyeyB73/5RBw+AFBlsb\n", + "7G/NjsBERGRI/Z1473GDeL/yibllKwEPAce0+Gw3fcfsPdjdEREnDxD/iNSr+XqgxbEayus/04m9\n", + "OT4CbAnMIfarslNhvYuIA5SUMYMrDT5usJ3F1saiNiqOlw9qp0REhsBQVRrbAQ8TH4F/CDggm78r\n", + "sdLYuLD+fOCSFt/VcscNdjY43GADg2kG7yhUHJ/pbTeSE7wDJCR4B0hI8A6QkOAdICE9VRpV9z11\n", + "C7GDrjWIj9CfD+w1xWcm28EFwOLs/RJid8OLGvFR+1WJrZi7gbN2g3tOhW+GuO47F8I7/wBfPgaO\n", + "asRtZItYlP3U9HBOM8XyOk3PTSyP5/TcxPJUOR2AI7PpxQy5y4g9PG5G+empi7PlZTquLQ1WN/hZ\n", + "ySmrrxocbcuP0SsiMmqG6vRU0eXAl7P3f2X5C+EPEi+Ul+l6xw3WnOSax83Zz3Oyn4stDo4iIjIK\n", + "hqbS+Chx8JDZxGsbpxH7jX9ptvx44immg4Btiaeu7gRmtfi+nnfcYMM2LpyPv+6yOMxiioJ3gIQE\n", + "7wAJCd4BEhK8AyRkaCqNc4nn0x4jDpZ+Kc0KY9zJxBbHMqZ+uK9vO24wy+D5Bu8x+MkUlccuBnMM\n", + "9rN0xiMJ3gESErwDJCR4B0hI8A6QkKGpNPqtsh03WG+SSuRlVeUQEekDVRoVbWyHSSqONarMIiLS\n", + "A1UaFW6wYbCKwY4GLy9UHB8xWLHqTJngtN0UBe8ACQneARISvAMkRJWGY4APlrQ6Wg3yPkjBYZup\n", + "Ct4BEhK8AyQkeAdIiHvZ6SWJHTd4e6HSeNLgld65RERaSKLs9JDMjhvMNDjUJvZ9dZh3LhGREsmU\n", + "nVVLbsezyiPf6qiq6/lQ0XaGQfAOkJDgHSAhwTtAQnoqOzVyXx814Ali54vjri4+y5E9E/Ici73z\n", + "XmvwMoNg8H6DE2yS8UNERLw1vAP0wEg0v8WHFOflZs0jDiq1HZ09VX4p8HPgJmA14oOPAfg34FXZ\n", + "d97YiA9Dioi0I9myc9CSOz2Vl92SO9ZBNyX/Y3BbB+vnX7dYrFRERKaSdNk5SMnvuMFOBucZXGhw\n", + "g8E3DF5vsE62vGHwEou9/GIw3eCVBgsMbjL4T4NLDL5nsePE2w3+UKw0FjbfX6rKQ+euc4J3gIQE\n", + "7wAJSb7sHJTa7jj8X0+9+xr8YuHESuRr3tmcBe8ACQneARISvAMkpLZlZ213vMhgC4PjcxXHjd6Z\n", + "RCRZtS07a7vjrWTXRcYrjm8avMrgSIPnemcTkWTUtuys7Y6XCPB/10QemeKi+S7OWQcteAdISPAO\n", + "kJDgHSAhek5DokYc1Gpt4D8mWe3nBhtZTW+5E5H6UktjElmrY+PsYcJjSloc37Q4pK6I1Etty87a\n", + "7ni3DA4pVBx/MXiWdy4RqVRty87a7niJ0O6KBqsZ/CZXcbxjgLk8BO8ACQneARISvAMkRNc0pH0N\n", + "WArsQLM7k89mlcehjrFERAZOLY0eGKyaPWGeP131TO9cIjJwtS07a7vj/ZJVHJ/OVRpLDTb2ziUi\n", + "AzU0ZeeJwLXAg8TeWb8PbFNYZwEwVnhd0+L7hmbHKxB6+bDBKwotjnOyp8xnTP3p5ATvAAkJ3gES\n", + "ErwDJGRormnsCXwWeBGwN/AU8GNgrdw6BlwGrJ977V9hxlpqxAp8NvCPbNZbgN8Dn/fKJCJSNItY\n", + "cRyQm7cAuLDNz6ul0WcGqxjsYPBYoeXxBYM1vPOJSF8Mbdm5AfH0U36ku3OJf+3eS/xL9xyybsRL\n", + "DO2Op85gRYOrSh4I/LnBXQa/N1jTO6eIdGVoy84LgOuY2J3Fa4EDidc6DgRuIPbYOrPk80O74wMQ\n", + "+v2FBisZHG1wWYs+rI7u9zb7JHgHSEjwDpCQ4B0gIUNZdn4KuJN4Hn0yGxDH3T6oZJkRT2fNy17H\n", + "MfEXI9RoOkyxvKdpg0MugXs+Dp80mGfZwE/fgesNVhjA/vQyHaZYXqfp4xLL4zld9/JhAc3ycugq\n", + "jTOAu4At2lz/VuB9JfOHbsdHgcFMi6MJjrc4Ds8tm2PwDoNDPDOKyKSGquz8NPBXYMs2118HeBx4\n", + "Q8myodrxUWJxmNozCqerbi1Mn551lriJd14RmWBoys6ziM9o7MXEW2pnZctnAZ8EXkg8bRWAnwF/\n", + "ya2TNzQ7XoFQ9QaziuPyFtc7iq9vWTYOegVCRdsZBsE7QEKCd4CEDE3ZOUYc76H48N4Hs+UrAZcQ\n", + "75x6HFgMzAc2avF9Q7PjFQgeGzVYz+CXBk8ZvDi762qtFhXHgwbrVhArVLCNYRG8AyQkeAdISG3L\n", + "ztrueOoMtjGYn100v7lQebxt6m8QkQEaaNn5LGDbwry9gYXAL4hdg3hRpTEkDHYrVBxXGOzknUuk\n", + "pgZadn4T+Ghu+lnAw8BviV1PPAm8a5ABJqFKoyl4B5iKwZtLTlmta3GEwbX7uKnQx+8adsE7QEKC\n", + "d4CEDLTsvB3YIzd9IvAnmvfmvxe4fpABJqFKoyl4B2iHweoGn2lxzeN8g+l92Ezow3eMiuAdICHB\n", + "O0BCBlp2LmPicKA/It7hNG5LYMkgA0xClcaQMvj3FhXHcVN/WkR6NNCy827iKG8Qu/t4ADg4t3wL\n", + "4ukqD6o0hpjBbIOTDF5p8PlC5XG3wUu8M4qMqIGWnd8DziOOq3AYsUuP/PnnA4CbBxlgEqo0moJ3\n", + "gF4YbFXS6vhJl18X+pltyAXvAAkJ3gESMtCyc3vgb8QuzMeAUwvLzwM+N8gAk1Cl0RS8A/TKYGeD\n", + "nxnckKs4unkgMPQ72xAL3gESErwDJGTgZec6wKuIT2oXHUh1T/oWqdIYQQbTCi2Oad6ZREZMbcvO\n", + "2u74qDP4UqHi+I5N7EJfRLrXU9k51X/EV7e5gW/3EqJLhgqScQFY5Jyhb7IKYqww+yfAdsAtwP3A\n", + "lcBZjXjqNC8wQseiRwEdi3EBHYtxAy07i/1EtXp5UEujKXgH6LesH6tdsj6rWnWEeHLJR0PVWRMW\n", + "vAMkJHgHSMhAy867iB0HzgM2HeSGuqBKowYM9jW4zWChwX8a/Guu0lhs8AyLPe7OsdZDA4tI00DL\n", + "zunEi93fIT7odylxSNay4VerpkqjprJnPB4vaXk8bPA2gzW8M4okrLKyc11ityG/I96Geya+lYcq\n", + "jabgHaBqBs8rO2W1MP68x+CfvDMmIHgHSEjwDpCQysvOLYi93I7R347mOqVKoyl4B/BgsKbB4Qan\n", + "Zg8IHvV9+F2hIvlWje+8Ct4BEhK8AySkkrJzFvAm4CrgEeJDfaGKDU9ClYYsJ3vO45RCxfErg9P0\n", + "zIcIMOCyc3fi6HlLieNnHAusPsgNdkCVhrRkcQTBXxUqj9O9c4kkYKDPaYwRx+j+MvCbSTam5zR8\n", + "BXQP+rhA7lgY7Av8MLf8xY3YYq6DgH4vxgV0LMbpOQ1xP1WYklCcYbBl/g4rh0xegneAhATvAAmp\n", + "bdlZ2x2Xzhm8tHCq6qwaXyCXenMvO71ubXTfcRkuBjcXKo7fGDzXO5dIxVzKzo2Bk4Bbaf/01InA\n", + "tcCDwH3EMca3KVlvHvFJ9EeJt/Zu3eL7VGk0Be8ACQmtFhg83+Dc7Anz8YrjxwbrV5ivSsE7QEKC\n", + "d4CEVFZ2ziB2YPgDYidx1wPHA3Pa/PwlwBHESmBb4sXzu4G1cuucADwEHESsUL5OrEBWLfk+VRpN\n", + "wTtAQkI7K2V3Vz2cqzz2abHeFgYbGRyXrfeYxVvQMdjWYOc+Zu+34B0gIcE7QEIGXnZuBXwCuBe4\n", + "B/gYsdIoayV0Ylb2PQdk0w1iJXJibp2ViJXIMSWfV6UhPTE4tuSp8r0t+0PGYMOyp86z1/259+3+\n", + "4SSSgoGWnVcRC/ZLiWODz8jmP0nr00bt2oB4amvXbHpONr1TYb2LgAUln1elIT0zOLJFpVB8xmOy\n", + "1/u890OkAwMtO8eA/yXe657Xj0rjAuA6mnew7Jptb+PCevOJp7aKVGk0Be8ACQmdfqDkzqri6yGD\n", + "N1rsTXc/gysMzjD4f9nyP1lsFacmeAdISPAOkJCeys4ZUyzfGTga+BrxAvZ84NxeNpj5FLGS2J32\n", + "dqDVOguIXbcDLAFuoPkAT8h+arpe00yxfLnpBlwG7AVg8RTpez8LT68AK74VPtuIf9wEYE/ig4I/\n", + "BMKmsPLi2IHn1p+BT60D370/tsoHuX+dTM913n5K03MTy1PldACOzKYXU5GVgTdmAZ4ktgiOZ+JF\n", + "7HadQby4vUVhfqvTUxdTXlGppSHuDF7VonVyucHrvPOJlKi87Nwc+CjxL7InKD911Mqngb8CW5Ys\n", + "a2TLihfCHwTeUrK+Kg1xl52yurBFxfGExTFpRFLiVnbOAF4JfK/N9c8iVgB7Ee+LH3/Nyq1zPPE0\n", + "00HE23LPB+4srDNOlUZT8A6QkFD1Bg1mGOxu8MOSiuN0g1Wy9TY0+LDBDgavMHiTwVyDZw4oWhjQ\n", + "9w6j4B0gIUNTdo4BT7N8v1UfLKx3MrHFsQw93Neu4B0gIcFz41nL450GtxcqjzuyZzxaXWwfRJcm\n", + "YQDfOayCd4CE1LbsrO2OS/qyymNZB7ftvt07s9RGbcvO2u64DIes4ni3waEGrzHY3GAFgzdbdjeP\n", + "wTWFyuOUAbU6RMbVtuys7Y6XCN4BEhK8A3Qiux5yfUnLY4HBxdbb0+ahXzlHQPAOkJCeyk4Nfyni\n", + "qBF7XNgVeFdh0RHA/sBXTP9PZQgF4IW56aOAq4FzKO9MsApqacjIsdgT7902sW8rMzjMO5uMjErK\n", + "zhuIt9dCfMbiceBs4hCw/1VFgBKqNGQkZddCGrZ8h4qreGeTkVBJ2bmU5rnV9xM7EQTYhfh0twdV\n", + "Gk3BO0BCgneAfjLYv1Bx7NrBA4NhkNmGTPAOkJBKrmmM0eyn6iXAj7L39wLP6CWAiLTWiOPXLM7N\n", + "uhp4ymChLd/ljkgyfgJ8BTic2HXI5tn8PYHbnDKppSG1YLBii2c77jf90Sadq6Ts3A64kdgNyMm5\n", + "+WcBX60iQAlVGlIbBtMNVjOYafACg6Xjt+Z6Z5Oh41p2rgSs4LRtVRpNwTtAQoJ3gCoYhFyL4xsW\n", + "Rxx8rU3seTp45UtQ8A6QkErKzm2B55XM357eB2PqliqNpuAdICHBO0AVDKYZXNTitNWXs9VCyecO\n", + "tNhZ4usMVixZvqXBswed30HwDpCQSsrOa4BDS+YfRhwS1oMqDam17JTV2S0qjpCtM8PgIIP5Bn8s\n", + "eep8FYPVs3WPyi1LcSRC6Y/Kb7nN2xx4qIoAJVRpiAAG2xmsmf3MVwpXtKhQ2nn90fwe3JXBqqTs\n", + "XEIc+rVoZ2KF4kGVRlPwDpCQ4B3Ak8FmBrca2MLlK4IbLI7hcZBN3k37+OtX3vvTR8E7QEIqKTu/\n", + "D3yHiWOKr5DNu7iKACVUaTQF7wAJCd4BvBk80+AXWaVxh8EXrKTnXINtLPbAe5fBTQb7ZPPzp7xe\n", + "Z7BP2eeHTPAOkJBKys6tgL8Rn8k4D/hv4gNHf0MXwkWGWnZtZFpuekZJq2NHz4zSVz2Vne0+EX4L\n", + "8e6p/yE+TLQWseJ4HvC7XgKIiK8GPN2IvT6MTz8FnF5YbZENblhakUqopdEUvAMkJHgHSEjo9oMG\n", + "6xq8NbvDKt/iKLv1fhgE7wAJ6ansnDHJsh2BXxPH9Z6qaXp9LyFEJC0NuA/4vMEXiNcvX5ct+rLF\n", + "G2DGGvrDTQrGgHVz71u9nnZJp19YkcpYHKI23+L4sWlwqGE1sLJzNs1fitlTvDyo0hCpkMGJLW7N\n", + "Pd47m3Rk4GXnCsDHgU378F17EG/fvZPYSjmisHwBy7dkrmnxXao0moJ3gIQE7wAJCf38MoMVDD7b\n", + "ouJI/Zbc4B0gIQO/e+pJ4O3055diFnG0v3cDy1g+vAGXAevnXvv3Ybsi0qMGPNmAdxJ7grgJODO3\n", + "+N0+qSRV3wbe1OfvXEocnyNvAXBhm59XS0PEWaG1MdM7j7RlYHdP5f0YOI3Yq+0vgUcKy7/dS4gc\n", + "A3Ynjgi4BLgC+ABwf5++X0T6aw5wa/b+cuL/X5FJ754am+RzkylrabwWOBDYJvt5A3Hwp7K/YNTS\n", + "aAreARISvAMkJFSxEYOjc62NlQxWz54qb3cs8yoE7wAJqaSlUdWtdV/Pvb8JuA64HTiA2M9V0QKa\n", + "4ycvIVYyi7LpkP3UdL2mmWJ5nabnVrG9BnzxIjhhFmwe4Epg7qKsbLHYN91pe8GGi+IZg4HnaTE9\n", + "t+LtpTQdgCOz6cVU5HBKBmwhtgCKrYV2lbU0ytwKvK9kvloaIokwOMDgkTZ6zr3ZYBPvvDVXSdmZ\n", + "f9Av75n09/RU0TrA48AbSpap0hBJiMGmFgdy2iV7GHBJi4rjB95Za8610tgBeKCD75lFbCbOJV5M\n", + "Pyl7v0m27JPAC4kPDAbgZ8BfsmVFqjSagneAhATvAAkJ3gEM1jI4r6TieJ/Bxwx+auVnEvotVLCN\n", + "YTHQsvPG7DUG3JybvpHYu+0jwDc6+L7AxO5Hxt/PJw4veQnxzqnHiefe5gMbtfguVRpNwTtAQoJ3\n", + "gIQE7wB5BpsbnNqi9bFtbr3NDPbv8wODoY/fNex6Kjun+keZl/38ILGr5Pyttk8Qx9f4Vva+akb6\n", + "T6GKSIHF2+hPJZYfm+UWvYR4C+8XcvOmqWPEvquk7DyS9Aaa1y+SyJAyWCP7edgUF87fbeU34Uj3\n", + "Kis7VwYOAU4gDsIEsTuBZ1QVoECVRlPwDpCQ4B0gIcE7QDsMDpmi4vh1WY+6Bo2y50EM1jD4s8VR\n", + "RseFQe/HEKmk7Nyc5vCuTxGbkBAvXH+xigAlVGk0Be8ACQneARISvAO0y2CawcezSmI/g4NLKo81\n", + "c+u/0eDO3LILct+T/8wnDFaeNkTHogKVlJ0XEc8zTifeKjteaexBswuBqqnSEBlhBhtZHAgqXwl8\n", + "yuALLVoknzB42yQtliO99ykRlZSdDwBbZO/zlcZmwGNVBCihSkOkBgyuKqkAxgx+OMVpreNLPrO/\n", + "5bolMjjU4O8GuxW2OcfgFIPXGKxW/V4PVGWVxvgtccWWxr1VBCihSqMpeAdISPAOkJDgHaAfDLYs\n", + "qRB2ypZNN3ipwQUGV2anrD6cXetoWHwC3RZO/Ox1BsHgscJ3HpF9Z9mpsVHqwbeSsvN84jMT0Kw0\n", + "1iD2azK/xWcGTZVGU/AOkJDgHSAhwTtAP1l84vzjFjs07eRzu34Hrp+iVTL+2rnF/PMMLjJ486D2\n", + "r0KVlJ0bAX/IXk8C1wL/AG6h/EnxKqjSEJG2GDzP4GyDfQwuzVUGyyapQA4vm++9L31Q2T6sQhyI\n", + "6Szgc8DRxNtwvYzCP56IVCw7dXWUwc7Z9IoGl5RUECtYvEurOH9v733oUW3LztrueIngHSAhwTtA\n", + "QoJ3gISEyRZm1z9uzSqFnxjsks2fZvDyQqXxpUoSD05PZWe742kArEe8w2Bdln/Q5uxeQoiIeMq6\n", + "KplTMn8MuNDgOcDVxPJvH4NVG/BwxTGHymHAMmJHgvcAdxdeHtTSEJHKZHdqXZu1Nrweau6HSsrO\n", + "24FT6KxlMmiqNESkUgbb5S6e7+udp0uVlJ1LKGm6OVOl0RS8AyQkeAdISPAOkJDQry8yeG9WaVyR\n", + "TTcM9jBYvV/bGLCeys52x/4+nzhOt4hI3X0v+7lp9vNjxArkVJ841Wq3T/WViAdqGXEApicLy0/p\n", + "Z6g2GRpPQ0Qqlj0d/ndgVeLNQVfnFq/diM+wpaySsvNdxLsI7gN+S3P0vvH3HnR6SkRcGPw2O0V1\n", + "RslzHKn/MVtJ2Xkf8J4qNtQBVRpNwTtAQoJ3gIQE7wAJCf38MoMvFSqKHw3RA4CVXNOYTvM8nohI\n", + "3Z2Ye/8Plu92/YsWe9GordOJ44SnRC0NEXFjsNBit+q7Z9PF01R3WWFUwUT0VHa2e+7tbOD1xGsY\n", + "v6F5IbyRBXhXLyG6pAvhIuImqxBmNuINQhi8hfjk+CHA7Gy1tRrxkYWUVFJ2Lsq9FuZe49Pt2gP4\n", + "PnAn8cL6ESXrzAPuAh7NvnvrFt+llkZT8A6QkOAdICHBO0BCQlUbMtgr19rYsKrtdmCoys79iPcy\n", + "vxp4BDi8sPwE4CHgIGKf+V8nViCrlnzXUO34gAXvAAkJ3gESErwDJCRUuTGDm7JK4/lVbrdNQ1t2\n", + "LmVipdEg9mOVv8C0ErESOabk80O74yIy2gzOzSqNd2fTKZ1Kr+TuqSpsRuxJ99LcvMeAnwK7uiQS\n", + "EenO+AN/uxl8APiHwfaegUZBsaWxK/E6x8aF9eYDl5R8Xi2NpuAdICHBO0BCgneAhIQqN2awdcnd\n", + "VDdVmWESlY2n4anVTi4AFmfvlwA3EC/OQ/OXRNP1mmaK5XWanptYHs/puVVubyasd2lhpsFWBps0\n", + "4I5Bb78wHWg+R7KYIVZsacwhtjR2Kqx3MXBuyefV0hCRZOWua5jB+dnP92fjchxrzQ4PHaINp7IL\n", + "4X9l+QvhDxLvfy4a2h0XkdGXVQ7fMjjc4OCs0ngkf8rKL9rwmEVsJs4l3nJ7UvZ+k2z58cTTTAcB\n", + "2xK7ZL8z+1zRUO34gAXvAAkJ3gESErwDJCR4btxg85JrHKo02hCIp6DGgKdz7+fn1jmZ2OJYhh7u\n", + "a1fwDpCQ4B0gIcE7QEKC58YNVjD4fUnFsZFPnHqq7Y6LyPDJRvhbweD0XKVxhk+UeqrtjovIcDPY\n", + "Pqs0HjV4ZvWbr6fa7niJ4B0gIcE7QEKCd4CEBO8ARQaLsorjldVvunspPREuIlInv85+vsfKb/aR\n", + "PlNLQ0SGlsFm2ZgbZrDU4LUWn1erYNP1VNsdF5HRYLBlyR1Vg+7csLZlZ213vETwDpCQ4B0gIcE7\n", + "QEKCd4BWLI7+l680Xjf4TXZP1zRERHxtwcSukr6aWFfqI0MtDREZGVm/VOOtjUF2o17bsrO2Oy4i\n", + "o8dgFYMHskrjzYPdVPd0emo0BO8ACQneARISvAMkJHgHmEoDHgU+n01u4JllMqo0RETScVf2c2+D\n", + "fzb4rcHLXBONEJ2eEpGRkg3U9HjJbbjP7e9m6qm2Oy4io8tgW4NrSyqOvxls159N1FNtd7xE8A6Q\n", + "kOAdICHBO0BCgneAThjMMDjMYDeDX+UqjscMVuz967unaxoiIolpwFMN+FoDrgZekVu0InCsU6yh\n", + "p5aGiNSCwSm51sai3r+unmq74yJSPwZrG4xlF8o3NTjNYLXuvqqearvjJYJ3gIQE7wAJCd4BEhK8\n", + "A/SDwfV9GGdc1zRERGriUu8Aw0wtDRGpFYNtSm7FXbnzr6mn2u64iNSXwQcMPp+rND7U+VeMjnnA\n", + "WOH11xbrjtSO9yh4B0hI8A6QkOAdICHBO0C/FVobq3T20e6leE3jFmD93KsfT0CKiIyaBbn3W3iF\n", + "8DYPuLFRKnljAAAIqElEQVTNddXSEJHaMmgYXJm1NPbr7KPdS7GlMYfY0+OtwNeAzXzjiIikpxEL\n", + "/39kkzM9s3jaFzgY2BZ4CbAQuBtYu2RdtTSagneAhATvAAkJ3gESErwDDILBf+Wua+zZ/se6N6OX\n", + "Dw/AJbn3vwV+BtwGHAGcUbL+AmBx9n4JcAPNR+xD9lPT9ZpmiuV1mp6bWB7P6bmJ5enX9C+At2Yz\n", + "Fxms3oClhfUDcGQ2vZgauBw4q2S+WhoiUmsGaxS6UZ/f3sdG10rE01P/XrJspHdcRKRdhdtv32Ow\n", + "2+Srj45PAnsQL37vAlxEPO20Scm6I7XjPQreARISvAMkJHgHSEjwDjBIBu8qGbBpeuvVu5fa3VMb\n", + "Ee+YugX4FrAMeCFwh2coEZGUNeBMYAPimRmAZwA7GrzZ4BxLr6x3oZaGiEiOwVoGF2etjX/JtTx2\n", + "mbhaPdV2x0VEWjH4Q0mnhi+auEr31GQZDcE7QEKCd4CEBO8ACQneASr0nJJ5q/bry1VpiIiMloNK\n", + "5vWt0hhmOj0lIlJgMLfk9NSrJ67SPbU0RERGy1jJvL6V9ao0RkPwDpCQ4B0gIcE7QEKCd4AKPVgy\n", + "T5WGiIiUKhu4bv3KUyRI1zREREqUXNP49MTF3VNLQ0Rk9LyIONbG+dn0sx2zJEMtjabgHSAhwTtA\n", + "QoJ3gIQE7wBVy0b2e17W0rh54qLuqaUhIjKCspH9bssmN1P/U2ppiIhMyeDerLWxcXNW91IbuU9E\n", + "RPrrNmBdIBhMb3incaSWRlPwDpCQ4B0gIcE7QEKCdwAvBj/MWhrftlhu6pqGiIi09FD2c5dJ16oB\n", + "tTRERKZg8IX8MxuopSEiIpP4ez+/TJXGaAjeARISvAMkJHgHSEjwDuDo3tz7p3v9MlUaIiKjLV9p\n", + "3OqWIgG6piEiMgWDfXPXNC5gRK9pvJ14b/Ey4JfA7r5xRESG1rLc+6vcUgzQa4EngDcDWwJnAkuB\n", + "TQrrqaXRFLwDJCR4B0hI8A6QkOAdwIvBbrmWxhaMYNn5v8DnC/P+AHykMG/kdrwHx3kHSIiORZOO\n", + "RVNtj4XBVrnbbbNZ3Uvt9NRMYEfg0sL8S4Fdq48zNNb0DpAQHYsmHYum2h6LBtwCHEg8c9Oz1Pqe\n", + "eiYwnYlX+wHuQyNPiYh0pQEX9+u7UmtpSHdmewdIyGzvAAmZ7R0gIbO9A4yK1Do8nAk8AhwKfCs3\n", + "/yxga2Cv3Lw/odGoREQ69Wdgc+8Q/fRzyi+Ef9ghi4iIJO41wOPEW26fSxwQ/SGWv+VWREQEgLcR\n", + "H+57DLgWPdwnIiIiIiL90mlXItsBVwCPAncCJw00XbU6ORYB+B7wV+JNBb8Gjhpwvip128XMc4g9\n", + "CywdUC4P3RyL44j37T9G/B05bWDpqtXpsdifeP30IeB+4LvE35FhtgfwfWL5NwYc0cZnRqbcbLcr\n", + "kXGrA/cA5xPvsno18ZfhPQNPOnidHosTgVOAFxFvMzwWeBI4bNBBK9DpsRg3E7gOuIjmKGbDrptj\n", + "8Sng98DLib8b2wP7DjRlNTo9Fptn638UmEM8DpcAfxx40sHaDziVWP49Ahw+xfojVW6225XIuLcB\n", + "S4AVc/M+QKw5h12nx6LM14Fv9i2Rn26PxRnAl4h/eY1KS6PTY7ElsaDsy1PBien0WBwMPMXERw72\n", + "Iv51vnbf0/lYytSVRlflZooP93XTlciLgCuJd13l198Q2LTfASvUr25V1gAe6FcoJ90eiwOy17+Q\n", + "3nNJ3ermWLySOJbC/tnP24AFwDqDiViZbo7F1cDDwFuIPVCsBhwJ/ILh/3/Sia7KzRQrjW66Elm/\n", + "ZP17c8uGVT+6VTkQ2Bs4p4+5PHRzLDYk7vfriedsR0U3x2IOsSB4DfEv0DcCWwEXMtyVaTfH4m5i\n", + "5Xkq8drOEmAb4mm7Oumq3Eyx0uiGerwttxvwVeJf2b90zuLhPOBzxNu2624a8TTEG4ljKlyVvX8B\n", + "sLNjLg9ziBe+zyXueyCezrmA4a5AO9VVuZlipfE34ji26xXmr0f8C6HMPSxfM66XWzasujkW43YH\n", + "fkC8G6J4vncYdXMs9gJOJt4I8CTwRWBW9v7owcSsRDfH4m7iefw/5eb9KfueZ/U7YIW6ORZvBe4A\n", + "TiDeXXgl8AZgT+Ipm7roqtxMsdJ4gniny8sK818KXNPiMz8DXszECzovBe4Cbu93wAp1cywg3nr3\n", + "A2KBeeZgolWum2OxLfHOmPHXB4m3ZG7PcN8Y0M2xuIrYq/Wc3Lw5xFM7dfs/0iBe9M4bn06xTByU\n", + "kSo3p+pK5DTgx7n1Vyf+VfE14rnJfwYeBP61oryD1OmxCMTb7T5G/Kth/ew17Bc8ofNjUXQko3P3\n", + "VKfHokE8RbkImAvsQLw/f7I/PoZFp8did2Lr5CTisxk7Em+5XQysXEniwZhF/LedSywDTsre16bc\n", + "nKwrkXOJd4DkbUv8T7CMWFMO7UMqJTo5FucS/0OMFV7F4zWsOv29yDuS0XlOAzo/FusTz9s/RLzg\n", + "eR6j8ccEdH4sDiZWokuJx+K7xBsDhlmg+f89XwbMz5bXrdwUERERERERERERERERERERERERERER\n", + "EREREREREenMbOITujs65xDpWp065xJJRZ263xYRqb1/I44n/Rixi+2P0LoVMUbsCG78ff51eQVZ\n", + "RfpqhncAkSFzGnAssSfQK4gjx+1IewPavIA4pOg+xHEcnhhQRhERScCqxN5AjylZNpupWxqt1hEZ\n", + "GrqmIdK+rYkD1vzEO4iIF1UaIv0xPvJb/iL3Ch5BRAZJlYZI+24mjhD3TyXL7s9+bpibN7ewzvg1\n", + "jOl9ziUiIon6KPAAcRTAZxMvbh+bLbuGOBb31sCuxAvl+WsaM2gOw7kesEZVoUVExEcDOAH4M7HV\n", + "8RfgQ9myrYiVxiPEu6N2Z2KlAXEc69uBp9AttyIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIi\n", + "IiLSN/8fdjQaZ3PTjvgAAAAASUVORK5CYII=\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "probs_rand = numpy.ndarray((1000, 2))\n", + "probs_rand[:, 1] = numpy.random.random(1000)\n", + "probs_rand[:, 0] = 1 - probs_rand[:, 1]\n", + "labels_rand = numpy.random.randint(0, high=2, size=1000)\n", + "\n", + "optimal_AMS.plot_vs_cut(labels_rand, probs_rand)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### The best quality" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "30.575776003303893" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "optimal_AMS(labels_rand, probs_rand)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Hyperparameters optimization algorithms\n", + "\n", + "__AbstractParameterGenerator__ is an abstract class to generate new points, where the scorer function will be computed. It is used in grid search to get new set of parameters to train classifier.\n", + "\n", + "__Properties__:\n", + "\n", + "* `best_params_` - return the best grid point\n", + "\n", + "* `best_score_` - return the best quality\n", + "\n", + "* `print_results(self, reorder=True)` - print all points with corresponding quality\n", + "\n", + "\n", + "The following algorithms inherit from __AbstractParameterGenerator__:\n", + "\n", + "* `RandomParameterOptimizer` - generates random point in parameters space\n", + " \n", + "* `RegressionParameterOptimizer` - generate next point using regression algorithm, which was trained on previous results\n", + " \n", + "* `SubgridParameterOptimizer` - uses subgrids if grid is huge + annealing-like technique (details see in __REP__)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Grid search" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "__GridOptimalSearchCV__ implemets optimal search over specified parameter values for an estimator. Parameters to use it are:\n", + "\n", + "* estimator - object of type that implements the \"fit\" and \"predict\" methods\n", + "\n", + "* params_generator - generator of grid search algorithm (AbstractParameterGenerator)\n", + "\n", + "* scorer - which implement method __call__ with kwargs: \"base_estimator\", \"params\", \"X\", \"y\", \"sample_weight\"\n", + "\n", + "* Important members are \"fit\", \"fit_best_estimator\"" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from rep.metaml import GridOptimalSearchCV\n", + "from rep.metaml.gridsearch import RandomParameterOptimizer, FoldingScorer\n", + "from rep.estimators import SklearnClassifier\n", + "from sklearn.ensemble import AdaBoostClassifier\n", + "from collections import OrderedDict" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Grid search with folding scorer\n", + "\n", + "__FoldingScorer__ provides folding cross-validation for train dataset:\n", + "\n", + "* folds - `k`, number of folds (train on `k-1` fold, test on `1` fold)\n", + "* folds_check - number of times model will be tested\n", + "* score_function - function to calculate quality with interface \"function(y_true, proba, sample_weight=None)\"\n", + "\n", + "__NOTE__: if fold_checks > 1, the quality is averaged over tests.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# define grid parameters\n", + "grid_param = OrderedDict()\n", + "grid_param['n_estimators'] = [30, 50]\n", + "grid_param['learning_rate'] = [0.2, 0.1, 0.05]\n", + "\n", + "# use random hyperparameter optimization algorithm \n", + "generator = RandomParameterOptimizer(grid_param)\n", + "# define folding scorer\n", + "scorer = FoldingScorer(optimal_AMS, folds=6, fold_checks=4)\n", + "\n", + "grid_sk = GridOptimalSearchCV(SklearnClassifier(AdaBoostClassifier(), features=features), generator, scorer)\n", + "grid_sk.fit(data, labels)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Print best parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "OrderedDict([('n_estimators', 50), ('learning_rate', 0.2)])" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "grid_sk.generator.best_params_" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Print all qualities for used parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "58.674: n_estimators=50, learning_rate=0.2\n", + "56.484: n_estimators=30, learning_rate=0.2\n", + "56.100: n_estimators=50, learning_rate=0.1\n", + "53.704: n_estimators=30, learning_rate=0.1\n", + "53.278: n_estimators=50, learning_rate=0.05\n", + "53.091: n_estimators=30, learning_rate=0.05\n" + ] + } + ], + "source": [ + "grid_sk.generator.print_results()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Grid search with user-defined scorer\n", + "\n", + "You can define your own scorer with specific logic by simple way. Scorer must have just the following:\n", + "\n", + "* scorer(base_estimator, params, X, y, sample_weight)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Define scorer, which will be train model on all dataset and test it on the pre-defined dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from sklearn import clone\n", + "def generate_scorer(test, test_labels, test_weight=None):\n", + " \"\"\" Generate scorer which calculate metric on fixed test dataset \"\"\"\n", + " def custom(base_estimator, params, X, y, sample_weight=None):\n", + " cl = clone(base_estimator)\n", + " cl.set_params(**params)\n", + " cl.fit(X, y)\n", + " res = optimal_AMS(test_labels, cl.predict_proba(test), sample_weight)\n", + " return res\n", + " return custom" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# define grid parameters\n", + "grid_param = OrderedDict()\n", + "grid_param['n_estimators'] = [30, 50]\n", + "grid_param['learning_rate'] = [0.2, 0.1, 0.05]\n", + "grid_param['features'] = [features[:5], features[:8]]\n", + "\n", + "# define random hyperparameter optimization algorithm \n", + "generator = RandomParameterOptimizer(grid_param)\n", + "# define specific scorer\n", + "scorer = generate_scorer(test_data, test_labels)\n", + "\n", + "grid = GridOptimalSearchCV(SklearnClassifier(clf=AdaBoostClassifier(), features=features), generator, scorer)\n", + "grid.fit(train_data, train_labels)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "(14265, 4755)" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(train_data), len(test_data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Print all tried combinations of parameters and quality" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "60.269: n_estimators=50, learning_rate=0.2, features=['fLength', 'fDist', 'fConc', 'fWidth', 'fM3Long', 'fSize', 'fConc1', 'fAlpha']\n", + "57.120: n_estimators=30, learning_rate=0.2, features=['fLength', 'fDist', 'fConc', 'fWidth', 'fM3Long', 'fSize', 'fConc1', 'fAlpha']\n", + "56.238: n_estimators=50, learning_rate=0.1, features=['fLength', 'fDist', 'fConc', 'fWidth', 'fM3Long', 'fSize', 'fConc1', 'fAlpha']\n", + "55.923: n_estimators=50, learning_rate=0.05, features=['fLength', 'fDist', 'fConc', 'fWidth', 'fM3Long', 'fSize', 'fConc1', 'fAlpha']\n", + "55.677: n_estimators=30, learning_rate=0.1, features=['fLength', 'fDist', 'fConc', 'fWidth', 'fM3Long', 'fSize', 'fConc1', 'fAlpha']\n", + "55.221: n_estimators=30, learning_rate=0.05, features=['fLength', 'fDist', 'fConc', 'fWidth', 'fM3Long', 'fSize', 'fConc1', 'fAlpha']\n", + "37.438: n_estimators=30, learning_rate=0.2, features=['fLength', 'fDist', 'fConc', 'fWidth', 'fM3Long']\n", + "37.281: n_estimators=50, learning_rate=0.05, features=['fLength', 'fDist', 'fConc', 'fWidth', 'fM3Long']\n", + "37.218: n_estimators=50, learning_rate=0.1, features=['fLength', 'fDist', 'fConc', 'fWidth', 'fM3Long']\n", + "37.063: n_estimators=30, learning_rate=0.05, features=['fLength', 'fDist', 'fConc', 'fWidth', 'fM3Long']\n" + ] + } + ], + "source": [ + "grid.generator.print_results()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Results comparison " + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "from rep.report import ClassificationReport\n", + "from rep.data.storage import LabeledDataStorage\n", + "\n", + "lds = LabeledDataStorage(test_data, test_labels)\n", + "classifiers = {'grid_fold': grid_sk.fit_best_estimator(train_data[features], train_labels),\n", + " 'grid_test_dataset': grid.fit_best_estimator(train_data[features], train_labels) }\n", + "\n", + "report = ClassificationReport(classifiers, lds)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ROCs" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": [ + "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEiCAYAAADTSFSPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n", + "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3XeYXFX5wPHvbMvupmx6IwkpkAAhEKpICQcUFBSQIqgo\n", + "RBQpSrFgFxEQREElCjYg8YdUQQEbIsgiRbr0mkp6IZvsZrN9zu+P9w5zd3Zmd2Z35p4zu+/neebZ\n", + "nTt37n33ZnLeOeWeA0oppZRSSimllFJKKaWUUkoppZRSSimllFJKKaWUUkopVVwWAfHg0QasAn4P\n", + "TEiz7/uA+4B3gWbgdeBiYFCafecCdwBrg33fBhYCu+c1eqWUUs4tBP4JjAUmAkcA7wD/StnvWKAV\n", + "uBFJEpOBTyCJ5xGgPLTvR4EW4C/AB4AdgX2AK4C/FujvyKQs4vMppdSAswipZYRdAzSEnlcDG4E/\n", + "pXn/XkAH8LWUfe/JcL5hPcTzVaT20gysRJIPwFSk1rR3yv5x4ISUfT4B/BvYDpwX/PxoyvuORBLj\n", + "6OD5DsDtwObg8Vdgp9D+k4F7kdpYI1IbO6WHv0UppQaERUjNIWE68CpSECccjxTQB2Q4xgPA81nu\n", + "250rgTpgPjAN2A84K3htKtknkmXBth2RBHEHcFvK+35PsnZUDbwF3IQ0vc0EfgcsByqDff6C1Nzm\n", + "BMf9UPBQSqkBbxHSN9KAfHOPI4XmyNA+3wi212Q4xrXIt3SAr/ewbyZDgCbgCxlen0r2ieTLKfsc\n", + "E8Q3JHheBWxFai4AZyCJJKwU2AScFDx/EekPUiovSlwHoFSePQLsCewP/AI4FBiX4zFs8DPWyxh2\n", + "QzrtH+rl+8OeTXl+P5Ikjw+eH4vEmWh+2wepATWEHluA4cCMYJ9rge8CTwCX0TWhKZUTTSSqv2kC\n", + "lgKvABcgBfG1odffDH7OzvD+3Uh+o38rtC2f4sHPcKIqT7cjydpRQhtwJ3Bq8PxUpL+nOXheAryA\n", + "JNPwYybw22Cfm5BkszDY/gTw/V78HUop1e8sonMfCUiNJA7sGzxPdKD/Oc3790Y6278a2ncD0jGd\n", + "zvAM24ciCe2sDK9XBTEdE9q2H+mbttLVFt6PJJRdkRFlHwy99nmkbyaX5rivA6tz2F8ppfqtRXRN\n", + "JCC1kjtDzz9G5+G/U0gO/62l8zDbY5HC+q9IgT0VKdwvo/vhvz9CRkzNR5qU9gfODr3+BPAYUts5\n", + "EGmSyzaRACxGah6r6FyzqQLeCP6OeUjNYx5wNcmRW9cinevTg7//YWSQgVJFYR4yPHMV8p/k9Cze\n", + "Mwf5T7Y9eN/3ChadKnYL6Tr8F+CTyDf4aaFtByBJZzOdb0isSPP+vZFEtC7YdzFwC9JklEkM6dhf\n", + "giSid5Dkk7ALkkgakc7vg+maSDrInEh+ELx+dZrXxiLNV+uDeJcCN5AcdLAAabZrQmpct5L+pk2l\n", + "vHQUcDlwIvIf6LQe9h+G/Oe9HfnmdiJQD3ylgDEqpZQqEg30nEjOQUachKet+A5SM1FKKeUB30dt\n", + "vR94FGkaSHgAmf5iRycRKaWU6sT3RDIeaecNWx96TSmllGO+JxLb8y5KKaVc8n020XV0rXmMC73W\n", + "SQljbZwNBQ9KKaX6mSV0ntgzJ74nkv8CVyGd7Yl+kiOQm6dWpO4cZwMWKn7OBbusYeKEDYw99xEO\n", + "/dAOrG5YyeQx7+S5W6WMtm3ltNUNoqVuEC11VTTVDaZxyzDqN4+gbksNWze8wNwXXme3ZRCLOsNd\n", + "EjyUXouwS9BrkXAJei0S+tT6E3UiGQzsHPxegnSYz0Wms16JzJi6H8k7dW9Fpm5YhAwbnoWMzb8k\n", + "0wli0AbXvgy8THCT1XLAQmkthx78H+YdsoxpoypobQeooLV8GPXDh7Bt+GAah1fSPHwQLcPLaB/R\n", + "wNBhGxnDRsawgbGdfm5kDO2UD2mnfEgT1ZN7+sOHszk+jPq2wTS2DKaxqYqmpkqat7ZTtjpOyfo4\n", + "JetWMentLQxfP4iWuqE0bFnLhPXbGLoVYh25XOTA1F68p7+a6joAj0x1HYBHproOoL+IOpHsR3JK\n", + "b4vcVPUDJFGcgTRjTQ/tX4/UQK5D7k7ejNyA9bNcTxyDDnjkEXlkx8r8R2ODuDo9OigZv4Gxk9Yw\n", + "sWod48vXMb58A2PLNjKmfCNjylupGLeO8fF3GVWyjGlsYUTJFkYMQmpX4XUs0t7Uti50f9hQtsaH\n", + "0tAxhG1tQ9jWMoRtzdVsbxpMY+MQtjVUs72hiqb6ViqaW6loqWdY/D9cNPtTXHDeVmqaP8pfXz6W\n", + "v6xF7t1pBJpj2v+klMqT3s5u6iuLZ3+ThcrFzBj9GAdP2ciYHRoYOn471eObqBpVQnxaO2Xjmqms\n", + "STy2UhOrYwRbGB7bwnC2UoPt1ZiIWsC892wEm5nKcnZkBTuywk5iVeskVjVN4Z1tw6iPtVP2UpyS\n", + "5R2ULmlg6GttlL+7lZpXT+GPqZMGFiODXBCl1yLMoNcioU9lp1eFbh54l0h6y0rT34S1jC99lENG\n", + "r2LSuM2MHLeNIWO3Uz1qO9Wjmqga3kTV8GYqh7ZSMbiM9rJXmb3DSDa3VtAaa2DooBVMrR5Es22h\n", + "slfXZTDb4rN4c1sZ7baCVltGuy2nLfEzXk7be88raI2X0R5fxrQNVTTdN4///HkMG7fsxmsb9+U5\n", + "rQUp5S9NJCH9JpHkgeG9b1s2hjTR7QhMLaNtWjltM4BdSohPrqR5aAelJWW0V7dTFotTUlJPTV6b\n", + "PStpoprtVLOdKppsJc3xQbS0V9LcUUVTvJrtNoatb2Tw+iaqtrVS0VZCfOtoNi3dgdVvD6VhdRVN\n", + "a0rpWP99Ll0XS07FnuO1GPAMei0SDHotEvpUdvo+akvlRcwiN3KuB55uB9pDr6Zvu7KxY7jvg1NZ\n", + "Pr2N8tJWKspaGFTeSkVZG+Wl7ZSVtVFe1kZ5eTtlZe2UlXVQWmqJzWqiak4DQ8dspaakmcqSJqpp\n", + "popmqtjMKJAPbGnwCBuKLCfbrUv4AcdwX/uuvF4Ww678AA/VHcGDVwCPxmBNzpdHKdUn/e3bu9ZI\n", + "vGRLjuSfg7dSM6SOEdUlxIeMZcOooTSMAoY2UVXZRvng8aybOpYNM8tpq6igtbKeYcPeYcrwVUwa\n", + "uo0h5esZV97IkIz/viUkB7dZYjaGjVli8RLibaV0tJbS0VZGe+t8Ft24gAt+EpMlapVS2rTViSaS\n", + "fs+OKqPt2HGsnzGUhsnDqN/zZebs0kT1oJ7fmzSUej7DzU8OYdvjM1jy8h68tPgAnloKbJARfkoN\n", + "KJpIQjSRJBkGTPuvjQElGxgzrZrtM0uIDymn7VxLrKWD0m23Uzb2A4yoW8f4ivP4xQFPcUDa1QOH\n", + "0MAE1jKGjW2jeLdxBHX1w6jfUEXTktm82nI3J966F//7z6Vc0hTxH5hPhgHzueiRQa9FgiaSEE0k\n", + "SQb9T5JgCF2LKSyfvTNvn9dB6cGbGD1hMyOr3mVUZbYj2yayumMc67dNZfnaiaxZ2k7Zqh1Y/foh\n", + "PPq64ZFnY3KDra8M+rlIMOi1SNBEEqKJRPWSjQE141g3eTpLdxnJ5jlTeOeIFexYtonR0+oYMext\n", + "Zpb3dJRRbGIGS9r35MX1I6hbsROLn5nBkr8fzsPPA+/qEGjlKU0kIZpIVAHZ8sN5aHgM++F3GfXh\n", + "SprHV7N9eBvlo9cyYcQqJg1upqrL3aOltHMxl3Iid29eyvSvHstffh+MpFPKF1p2huh/ziTjOgCP\n", + "mGhOY2OVbJ+4N8+evhfP3TKHF58bw/oGsDb8qGZb+yxe37I/Tz47hvU/APv+aOID9HMRZlwH4BEt\n", + "O0P0YiQZ1wF4xLg7tR0E9p4YHW/O4nU7hvU2NbGEHocGTWyFZAp8/GJiXAfgkT6Vndl+aEuBfYGZ\n", + "QA2yjvrbwHN0vrfNNa2eKW9ZqABmPsThU59m/31HUPed59in7AbOfG+fEWxe3E7ZXQ0MexOohdhy\n", + "R+GqgaWgZed+yMy8DciUFFuR6d63Bs+3Ba/vX6gAcqQ1ElU0LJRaOPHPHPfSp/k/O4imtDWVUtpu\n", + "BXsM2F3AVkRQa1EDT8HKzj8jU2r8EpnKfUTK6yOADyFTvK8H/lSoQHKgiSTJuA7AI8Z1AD2x8IGX\n", + "mX3Hsdyz/gj+Ge+m+cvG6Ogoo3X7IJrWV9C88ij+tncOpzKF+huKkHEdgEcKVnaeD1RmuW9VsL9r\n", + "mkiSjOsAPGJcB5ALCzUWvrGKiZv+wkfqLufbtpLtFqwtpyVtcvkHH/qvhdFZHN4UOv4iYlwH4JFI\n", + "+kiKhfaRqH7FSv+kAeYBk4BYM4OGNDB05OV8d9YCLpiU2HcaS/ksC1sms3LjLN7cWMPWN3fj9R/F\n", + "4EVH4aviEcl9JPcCNwB/I7fpu6OmiUQNKJVsn9hM1eru9hnB5vgQtrUPprFtMI2t1WxvHUxj0zz+\n", + "s/Lr/PiBUuLXx2T1UTVwRZJIbgE+hnSy/x64CRm15RtNJEkGnf4hwdCvr4WtAT48jK3jY9iZFbTu\n", + "u5GxGQbA1BJu0dmR5czhZYazZUMNW9dMZ+mrF3DtP0uJvwa8GPNrVGa+Gfr15yInkd3ZXgN8Cllb\n", + "fR/gMeBG4E7Al0nsNJEkGfQ/SYJhgF6LhZw+Mk7JrPWMG7OVmtH/5bn9ZnDUln9w1McaGDq5kSGD\n", + "U9+zM2/xCW7nk9zGrryxDvgy8G9gcz9LLIYB+rlIw0nZuTvwM6AZuafkN8BuUQeRhna2K5U1Wwp2\n", + "n8ms+PQ8aheMY+3SKhqbUjvy51Frz+E6ezsn21VMtC2UX2fhXAuTXf8FKm8iLzsnAt9BmrYakGau\n", + "fyDfVC6KOpgUmkiU6hNbBvaDYGszDT+uZpsdw3p7Htfax3n/KxY+4Tpq1WeRlJ0VwMdJJoyngTOB\n", + "IaF9jkVqJy5pIkkyrgPwiHEdgEdM9rvasWD3BPthsD/PlFj250l7Lr9s+DVfWNHA4Ets9rcNuGZc\n", + "B+CRSMrOTciojl8Ae2TYZwSwPIpguqGJJMm4DsAjxnUAHjF9e7uNgZ0H9mvpksp+PGWfZH9r4XHr\n", + "f3+lcR2ARyIpOz9DcXzL0ESiVGRsNdhjBtF03STe+cNgGjYmEspgGmwdNfe7jlBlLZKycyEwNM32\n", + "wUgfiS80kSjljB0B8efCNZRxrG2ezuIlu/Da/ZNZcWUJ7SeBnQs2XXmi3Imk7IwDY9NsHwN0RBFA\n", + "ljSRJBnXAXjEuA7AI6awh7elYG/cmTdbSmnrbsp8uxNvvXoJF8+T+2CcTERpHJzTV30qO8t6eH0k\n", + "yXbOkXQeQ14KfBSZsFEppYBYB/C5t+DMDYw54Fn2NauYtP96xu26gbGT32FK1SvszlJmsJidd7uE\n", + "HzySeGc12y7czuAHgXXAZl1Fsv+I9/DoAL7rLLqu9IOnlMcsjLPwges45+cH8IQdytYMNZZ4K9iV\n", + "YG+QIcmqwAo6aaMJfv4bOBGoC73WCqwAup3nJ2J6Z7tSRcLCBOCijYzeYwHnz36CA8evYSJrmcBW\n", + "hqfufhpwC8R8nuuvmEVSdk4FSgp9kjzQGkmScR2AR4zrADxiXAeQiYWdLdxs4dntVNqLuCpdbeVh\n", + "sJeAPTwPpzR5OEZ/UbCyc2+kHyTxe3cPX2giSTKuA/CIcR2AR4zrALJhYZqFxqVMtWfxK1tDXbqk\n", + "sg7sYrBvgf0x2FxvUTCFiL1IFazsDI/U6qmfxBeaSJTqRywMs/BTCxveZoa9nrO7Gwn2L0ejv/qD\n", + "gpWdU0k2Z03t4eELTSRK9VMW9rfwrQ5iS+qosSvZwd7FCfZI7g8nk+fAVrmOtQhp2RmiFyPJuA7A\n", + "I8Z1AB4xrgPoKwtVFk6x8G0LTRbsdZwTHvG1BuzpYHvq1zVRxFskIik7zwM+nWb7p4FzczzWucAy\n", + "ZA2TZ4GDe9j/aOBJoB7YCNwD7JxhX00kScZ1AB4xrgPwiHEdQL5ZqLbQ8hK7pzZ1LQO7SzfNXSbK\n", + "OD0XSdm5hPQF/iHA4hyOcwoybPhzwCxgATIVfaZ1DXYK9v8RMB3YE7ifzKszaiJRaoCy8Mk3mNmy\n", + "Oy+l3pPyFtjzXcfnuUjKzmbS94VMC17L1lPIIlhhbwFXZNj/JORu+vA3isOQTv6RafbXRKLUAGZh\n", + "BwvbHuZQexqL7Eg2WbA2Rod9ggOutMmRqKqzSMrO5cDxabafAKzK8hgVQBtyY2PYL8m83OUEZI2T\n", + "LyAfgKHImvFPZthfE0mScR2AR4zrADxiXAdQaBYGW/iahRtbKVt1CI+8N8W9lZnME4yrGD0USdl5\n", + "BbASOBIoDx4fCrZdleUxJiI1idQmsouBN7p534HABiQJdSD9KmMy7KuJJMm4DsAjxnUAHjGuA4iS\n", + "hdip3PzpRDPX3Rz/UqhWYlzG5pmCTpGSUIHUBE5BkgHI0OA7kakLWrM4xkSk9jIPeCy0/WLgU8Au\n", + "ad4zHal9LARuBYYBlwavHU7XP94GcS4Pnm8BXiBZ4zHBT32uz/X5AHpeyfa3m3lqJ4BTWcW+PNtS\n", + "zZ82r2Tl9ZfD5a7jc/DcAPOD58uB7xPh9FI7AycHj5k5vjdT09Z1wMMZ3nMV8FzKth2QZHZgmv21\n", + "RqKU6mIQTZPT3cT4SW6xCzl9+yomPtxOyZm2OKaCKoSiKjufJH1n+w8z7P9j4JmUbRNI30QGRXYx\n", + "Csy4DsAjxnUAHjGuA3DHTof4pUPZeg3E2+DhLonlT3xsoK7qWLCycwGyAiLIWu0L0jwS27N1MtCC\n", + "DP/dFbgWuT8kMfz3SuDB0P4HI/0i30NqQ3sjw3+XA+nuXtVEkmRcB+AR4zoAjxjXAfjBVsJ3vltC\n", + "+0NVNG4JJ5OXmb3AJsu+gaJgZWctvDeXcy3S/JT6SGzPxTnIDYnNSG0jXLNYCCxN2f8kpIO9AVlE\n", + "6x7S96eAJhKlVC/sxXMHJhLJHF60GxnVaGEP13FFqGBl544UX3uhJhKlVK+U0zJrKFu3gLUj2WQf\n", + "5tB2C9dZqHEdWwQKVnZ2kJz999/QdaUZD2kiSTKuA/CIcR2AR4zrADxium6yE0tpexOsHUST/Ty/\n", + "tQ9xWLOF7/TzhFKwsnMLMDv4PTylvM80kSQZ1wF4xLgOwCPGdQAeMek321Kwt4T7TS7hYmuzv/m6\n", + "GBWs7LwL6ZOoRRLJY6TvJ/l3oQLoBU0kSqk8sQeFk8kcXrSbGX6166gKpGA3JFYDZyATJ56PdIQ3\n", + "ZQjgvL4EkUcWXbNdKZU3diJwPXAcwJ84nuO5Z3Ks/9VOIik7a4ERhT5JHmiNJMm4DsAjxnUAHjGu\n", + "A/CIyXbHUtpuSNRMzub67VZuX+hPtOwM0YuRZFwH4BHjOgCPGNcBeMRkv6s9qIzWbYlkciE/tfdz\n", + "5IyCRRa9gjVtLQC+BTQiNx6mO1Es2O7LXP/atKWUKhAbixFvtpRUJLa0ULFgEG0XuIwqT/pUdpZ1\n", + "89oeyCy/AHPoPpEopVQ/F7MWO3YKKx5/hx1nA8zlhfMPpG7wExx0DsTaXEeo8kOTWpJxHYBHjOsA\n", + "PGJcB+AR09s3DqLprHGsbQ6N6NqwK69+HeyMbpb29VkkZWcF6ee2qgpe84UmkiTjOgCPGNcBeMS4\n", + "DsAjpi9v/ho/HvZLzl0xiKbUyR9Xg30J7Flgi+UmxkjKzvuAC9NsvxCZ+8oXmkiUUpGxEHuYQy9f\n", + "wJfsCdxlR7Gxy4zCYP8oMw97LZKycxPJu9zDZgMbowggS5pIlFKRshCz8HkL97dTUv8M+9if8NUu\n", + "CaWcllmuY+1GJGXndtLPuLsbMouvLzSRJBnXAXjEuA7AI8Z1AB4x+T6ghTILe1n4QytlK2/lE7ac\n", + "FgvWVrI9/k+OOCXf58yTSMrOp0gucRt2GTLFuy80kSQZ1wF4xLgOwCPGdQAeMYU8uIUSCwe9wm63\n", + "Jmol+/GUXcfY3Qt53l6KpOw8Glkm91ZkUarPAbcB7cAxUQSQJU0kSinvzOHFixPJ5DK+s9p1PGlE\n", + "VnZ+GHgcuUGxEZnE8aioTp4lTSRKKQ/ZikE0tSSSyavsep9NPxLWFS07Q/RiJBnXAXjEuA7AI8Z1\n", + "AB4xUZ5sL56bmkgkU1lqlzNlqZXlx30QWdlZBXwc+AbJRa52AkZFFUAWNJEkGdcBeMS4DsAjxnUA\n", + "HjFRn3ACqw8Nj+R6hd2shWOjjiONSMrOnYDlyDDgdiAxJvpq4IYoAsiSJhKllOfsOIivTSSTr/Fj\n", + "exUX7es6qChO8lfgd0Ap0EAykcwDlkYRQJY0kSilioD9YLhm8jH+ZC18z2VAUZxkMzAz+D2cSKah\n", + "95H4yrgOwCPGdQAeMa4D8Ihxe3pbM5fnn08kk1/zBdtGqatRsJElksTY59QayfooAsiSJpIk4zoA\n", + "jxjXAXjEuA7AI8Z1AGCrwzWTyayw/2PPi1wEEsVJbgduCn5PJJIaZOXEmzK8xwVNJEqpImMrK9n+\n", + "23BCGcu6J8BeAHZkVEFEcZIdgLeCRxvwDFAHvAGMjSKALGkiUUoVpQ/wr7Pm8GI8nFBGsmnjQxy2\n", + "QwSnj6zsrAbOAK4DfgV8Hr9uqAFNJGHGdQAeMa4D8IhxHYBHjOsAUlkY+Rq73HIDZ2yfxhIL1p7M\n", + "7Y228Cu/atkZohcjybgOwCPGdQAeMa4D8IhxHUAmFiqP4d6FYG0pbfYOPv79wp8yGvsANwPPIRM1\n", + "3hxs84kmEqVUv7E/T64M+kw6zub6gwt4qkjKzlOBDuBfyIy/lwW/dwCfiSKALGkiUUr1G7XM2+lg\n", + "/tMM1k7inY7bOOWTBTpVJGXncuDbabZ/K3jNF5pIkozrADxiXAfgEeM6AI8Y1wFk4zecOWV3XqoD\n", + "a8ex1i5h2iUF6DOJpOxsRKZJSbUzsuiVLzSRJBnXAXjEuA7AI8Z1AB4xrgPI1kzeGJkYyXUe11oL\n", + "J+b5FJGUnX8Dzkyz/fPAP6IIIEuaSJRS/ZR9AKytotG2UpauhahPB8/z8dL6IjJh42+A+cHjN8AG\n", + "4FzghNDDJU0kSql+yk5K1EoWcvqj+T54no+XVjyHh0uaSJKM6wA8YlwH4BHjOgCPGNcB5Gomb/wx\n", + "kUxWssPn83joois7zwWWAU3IMOJshrRdiNxF3wysAa7MsF/RXYwCMq4D8IhxHYBHjOsAPGJcB5A7\n", + "u38ikRzCI9bCnHwdOE/HyVl5L95zCtCKrPk+C1iAzN01uZv3/BR4E1kbfiqwJ7LsbzqaSJRS/Vo1\n", + "276eSCb3c2S+5jqMpOy8ADgp9Pwm5B6St5CEkK2nkL6VsLeAKzLsPwtJPNmeQxOJUqrf25VXXwRr\n", + "v8pPHsnTIftUdpZkud/5wMbg93nIkrufAv4HXJPlMSqAvYEHUrY/AByY4T3HIQtnHR38XAYsAsZk\n", + "ec6BzLgOwCPGdQAeMa4D8IhxHUBv7cmLrwA8zGHpbsuIXLaJZCLJlRCPAe4C7gAuAd6f5TFGIyss\n", + "pq5fsgEYn+E904EdgZOB05C76HcB/kLhJzFTSikv7c3z/wN4nn0mgj0bbFGUh+uBxJrCLwCfDn7f\n", + "GblZMRsTkVFdqZ3rFyMd6en8NnhPOOvuHGzbL83+2rSllOr3XmG3weHp5iH+KNg9+nDIPpWdZVnu\n", + "9wCyZvvzSKGeuAlxN6S5KRubkH6VcSnbxwFrM7xnLdAOLA5tWxwcZwqyLkqqRSSnbdmCJL7a4LkJ\n", + "fupzfa7P9XnRPt+d1xqvo+SmVXzgjBv5AxsYdzDUvgi/XgRnfzaL4xnkfkCIcJqrGuAXwL10HjF1\n", + "Kenn4MrkSdJ3tv8ww/5HILWP6aFtM4Jt+6bZX2skScZ1AB4xrgPwiHEdgEeM6wD6ysJH3mXEplO4\n", + "zYK1ZbTW9/5QxeNkoAUZ/rsrcC1QT3L475XAg6H9Y8i9JrXAXGAv4BHgiQzHL6qLUWDGdQAeMa4D\n", + "8IhxHYBHjOsA8sHCxDZKX080c1Wz7Xe9O0xhDCvQ/ucgzWHNSNNUuM9kIclO/YTxwJ1IwlmPrIOS\n", + "adSWJhKl1IBjYccFfKkZrB1MQzvYXO/zK1jZuQG4HGlK6s5OSNPUhkIFkgNNJEqpAamZil0raLZg\n", + "7XjW/CjHtxes7JwO/AFpinoV6cS+FPh68PP3wGvB63+gcz+GK5pIkozrADxiXAfgEeM6AI8Y1wHk\n", + "27n8ciNYuwMr1+T41oKXnWOR+bFuR/or3kKapG4LtqeOwnJJE0mScR2AR4zrADxiXAfgEeM6gHxb\n", + "w/gbg073ONhBObxVy84QvRhKqQHLQuVM3oiDtWdww/G5vbX3sr2zXSmllOdi0DyBtUsABtFymet4\n", + "ipXWSJKM6wA8YlwH4BHjOgCPGNcBFMKhPPypYPSWrWXeOVm+TcvOEL0YScZ1AB4xrgPwiHEdgEeM\n", + "6wAKZRdeex6sPYXbWm33y3QkaNkZohdDKTXgVdA8rZyWDrD2bxz1nO25G6NgZedNwNDg93n0biGr\n", + "qGkiUUopYDQbrgFr5/K8babiqz3sXrCysw2YEPweR4YB+04TSZJxHYBHjOsAPGJcB+AR4zqAwrLV\n", + "VTRuBGuv5+w2C7O727kvZ+quurMcOI/kxT4QqZmkeyillPJKbHsT1ecCfI/LytYx7osuojgOmfo9\n", + "3sOjw0VwGWiNRCml3mNjw9m8GKz9Fx+wNvOCgAWrkdyLrGo4Kng+G2neSn34dGe7Ukqp98RsC4Oe\n", + "A3iRPSG7EVx5Fe5sPxTtbC82xnUAHjGuA/CIcR2AR4zrAKJhLwVrd+MVa6WlKe1OhTp7uLO9A+1s\n", + "LzbGdQAeMa4D8IhxHYBHjOsAomGPTKxV0kjVikw7FersbwNXIBc7DnwM/zvbNZEopVQndhDEN4O1\n", + "3+KH1sLgdDsV6uza2a6UUv2CPQasPZDHrJXVZrvs0Jeja2d7/2VcB+AR4zoAjxjXAXjEuA4gQusA\n", + "WhgE6WskfVKWxT51wOHAYqTfRCmlVHFpBljMTgBVLgMZD1wE/AqpqYCstz7NWURdadOWUkp1Yacn\n", + "Otwf48BT0+0QRRT7AFuB/wGtJJfV/QFwaxQBZEkTiVJKdWFjiUTyce74Tbod+nL0bBe2uga4FtgL\n", + "WaM94X6kVqL8Y1wH4BHjOgCPGNcBeMS4DiA6MXss974F0MKgmnwfPdtEsjewKM32dWhnu1JKeW8Y\n", + "9dsAljNIfNPpAAAeIUlEQVT1ffk+draJpAkYmWb7LGBD/sJReVTrOgCP1LoOwCO1rgPwSK3rAKJU\n", + "zzALMJE1eT92tonkXuD7QGVo2zTgx8Dd+Q5KKaVUflXQ+mjws7KnfXOVbSK5CBgBbASqgceQ4cBb\n", + "gO/mOyiVF8Z1AB4xrgPwiHEdgEeM6wCitIGx6wDilDi5jwRkxNYhwGHICK4S4DngwXwHpJRSKv+2\n", + "UvMGwLuMqnYdi+90+K9SSqVRTstMsHY6i63telOilp0hejGUUiotOxisraDZNlNxYuqLTkLylF6M\n", + "JOM6AI8Y1wF4xLgOwCPGdQBRG8G7LWDt88y9LeWlSG5IVEopVeSq2f4cwBe57kjXsfhMayRKKZXB\n", + "Cdz1DbB2GkviKS9FXiMZjtycGH4opZTy3Mnc+SjAFobHXJx/KjKvVjO6sFWxMK4D8IhxHYBHjOsA\n", + "PGJcBxC1f2MGldJmwdoZvF0ReqlPZWe295HchNREzgDW9vWkSimlonc4tS0TWc8admA/npm1BF6O\n", + "8vzbgDl5Ota5wDJk/q5nyX724J2BhuCRiSY4pZTqxu681AzWns7C00ObI+kjWQ6yRmMfnQL8HLgc\n", + "WTf4CeAfwOQe3lcB3A48giYLpZTqtVG82wiwidE75+uY2SaS84ErkFpBX3wFWAjcCLwZHHctcE4P\n", + "77sKeAH4I+Ckk6gIGdcBeMS4DsAjxnUAHjGuA3Chhq2NAGW0521dkmz7SO5FaiRvIgtbtYdes8Cw\n", + "LI5Rgaxr8uOU7Q8AB3bzvo8Ej7nAyVnGq5RSKo2hNDQAtDBobL6OmW0iOS8P5xoNlALrU7ZvQNaD\n", + "T2ci8FvgY8D2PMQwkNS6DsAjta4D8Eit6wA8Uus6ABeG0rAFoImqMfk6ZraJZFG+Tpijm4FfAc84\n", + "Or9SSvUrw6jfDPAKu++Rr2Nmm0hAFrU6FdgVac56DbiVzmu4d2cTcs9J6tK845B+knQOA+Yhi2qB\n", + "9I+UAG1Iv8oNad6zCBkcALJeygskv3mY4OdAeJ743Zd4XD5PbPMlHpfP5yIDXnyJx+XzCxmA5UMl\n", + "ze9CLdu5djCdy8uC2w14BymYH0UWttoCrEASS7aeBH6Tsu0t4IfdnDf8+DbQGPw+PM3+OqIrybgO\n", + "wCPGdQAeMa4D8IhxHYALt3Py98HaiayqC22OpOz8F9LhHu5UHwbch3SWZ+tkpAbzOSQBXQvUkxz+\n", + "eyXdL5Y1H72PRCmleu1OTjoPrJ3A6nC/cyRl53Zg9zTb55B7J/g5yA2JzUjfR/iGxIXA0m7eOx9J\n", + "PJloIlFKqW78jAsOB2vl8Z5Iys7NpL8D/eDgNV9oIkkyrgPwiHEdgEeM6wA8YlwH4MIVfHNCMpHY\n", + "xHxbkZSdv0c61w9GhvCWImu4v4a7EV3paCJJMq4D8IhxHYBHjOsAPGJcB+CChYpQIhme3Fx4I5A+\n", + "kjhyM2J78PufSd/p7YomEqWU6oaF2FjWWbD2/Tw+Nbm597Id/lsHHIdMkZIYpfU68HZfTq6UUipa\n", + "MbCTaWkHyo7nz7v/Nw/Df3Nd2OptZKTWfWgS8Z1xHYBHjOsAPGJcB+AR4zoAV4ZR3wGwheEmH8fr\n", + "rkayAPgWct/GL0hf9YkF28/PRzBKKaUKbxWTSgE2MHZoPo7XXSLZAygPfp9D94lE+afWdQAeqXUd\n", + "gEdqXQfgkVrXAbiyP08v+xdH7txBaT6WB+k2kZgMvyullCpi5bS1AnRQWpmP42XbR3IxUJ1me1Xw\n", + "mvKPcR2AR4zrADxiXAfgEeM6AFfKaG8DeIuZU/JxvGwTySXAkDTbBwevKaWUKhJbqSkHGM+6SVGe\n", + "Nw6kWwTlg8DGKAPpgfbXKKVUD87hukVg7TjWNgebCnofSXiCxKUpJytFppb/dV8CUEopFa13mPIm\n", + "wFg2tKeuNNgbPSWSxMqINyFTuIcnTGxFbmR5Ig9xqPwzDOBRKSkMei0SDHotEgwD9FpMY9mafB6v\n", + "p0SyKPi5DEkYbfk8uVJKqehZYgDEsLF8HC/bzvZxwNFpth8HnJSPQFTe1boOwCO1rgPwSK3rADxS\n", + "6zoAVzoojefzeLmM2kq37sh2dNSWUkoVlTglThLJNGRJ3FSLgRn5C0flkXEdgEeM6wA8YlwH4BHj\n", + "OgBXEk1bQKRNW3XArDTbd6b7pW+VUkp5JtG0FcvTHRPZJpJ7gZ/SOZnsAvwMuCcvkah8q3UdgEdq\n", + "XQfgkVrXAXik1nUArrjqI/kmsBVZEXFV8Hg12HZRPgNSSilVWImmrXWML+9h16xkm0i2IsvsfhiZ\n", + "Xn4B8CHgoOA15R/jOgCPGNcBeMS4DsAjxnUAroygrgFgPePLwKabtWRA0ylSkozrADxiXAfgEeM6\n", + "AI8Y1wG4YmHn0LrtBxFR2RkDvog0ZzUB04Pt3wROjiKALGkiUUqpHljYaR61iURyKH0sO7Nt2roA\n", + "+C7wu5Tta4Av9SUApZRSkYuX0Z74vacZTnqUbSI5BzgT+Dkkzw48D+ze1yBUQRjXAXjEuA7AI8Z1\n", + "AB4xrgNwKJxI+tzhnm0imQK8nGZ7G7K4lVJKqeJhy5NTJ0ZWI1kG7JNm+1HIkGDln1rXAXik1nUA\n", + "Hql1HYBHal0H4FBem7ayPcBPgF8itY8S4EDgNODrwBl9DUIppVSkGiqRNa3GsW58PtYkydaZwDvI\n", + "aolx5KbEz0V4/mzoqK0k4zoAjxjXAXjEuA7AI8Z1AC59lhvfAWs/wl9upcArJIb9LniMQWolUSYx\n", + "pZRSebSd6tXA5EG0jHQdi2+0RqKUUlk4lZtrwdo5vPg6BayRvAzMQ2b+TTdiK2xbsM9lwMq+BKSU\n", + "UqrwhtKwDaCeYUP7eqzuEsndyLrsid+7Mwj4AHALknyUe4aBPSolzKDXIsGg1yLBMICvxR68tBqg\n", + "hHif1yTpLpFckuH3TGYgU6gopZTyXCkdmwBKiFf29VjZ3kcCMt/WvsApwJBg2xCSd0UuAcb3NSCV\n", + "N7WuA/BIresAPFLrOgCP1LoOwKV1jF8KUEb7oL4eK9tEMg74L/A0cCuQmHb4GuDq0H5bsjzeuchN\n", + "jk3As8gU9ZkYZGGtNUAj8CLw2SzPo5RSKo04JU0AHZTmUqFIK9sD/AzYAIwCtoe2/xFZlyQXpyBz\n", + "dl0OzAWeAP4BTM6w//uR5HEiMBv4FfBb4JM5nnegMa4D8IhxHYBHjOsAPGJcB+BYC0AHpXlZtz0b\n", + "60lOzthAchr56XROLNl4CvhNyra3gCtyOMYdwF1ptmcawrY5eE0f+ujrYzPFzbgOwCPGdQAu/YDv\n", + "HQPW7siyFuSz3WvZ3pBYBckZvkJGQ3CffXYqgL2BH6dsfwCZdiVbNchd9tkaAUSWdVW/1qf/cB6o\n", + "dR2AR2pdB+BSDNsMsIKpFX09VrZNW48C81O2lQHfAB7K4XyjgVK63hW/gew76j8KHI40bymllOqF\n", + "ubywKV/HyrZGchHwH2A/5J6Rq5Gmrhpk3faoHITcq3Ie0kmfziJgefD7FuCF0Gsm+Fk7AJ4nfvcl\n", + "HpfPE9vyebx8xhfl87lIH6Uv8bh8fiFSPvgST5TPzWn87UtSP5hKlCYAlwJ/A/6OdJZPyPEYFUgT\n", + "2Ykp268DHu7hvQcDW4Hzu9knU7NDsTdH9IZxHYBHTB6PVeyfJeM6AI8Y1wG4ZKGc99ZtL/znugLp\n", + "IJ+Vp+M9SfrO9h928555QD3yDaI7mkhUoelnSfULFmKltEWWSED6MGbm6VgnI8POPgfsClyLJInE\n", + "8N8rgQdD+xvk/pGrkPtZxgePMWmO3d8TySHAG928vgiZ76wY1OLfMgTZ6C+fJaUopyXSRHI1srhV\n", + "vpyD3JDYDDxD5xsSFwJLU553kFwHJfEI75PQ3xNJTxYizY+QudpuyN/EmrX0Phk8TPaLoi1HBlj0\n", + "lsnjeYr9s2RcB+AR4zoA1yrZnpdEkm1nezXwaeAI4DmkhgAypNbSfb9FOr8KHumk3rX+2TTbBqIy\n", + "SK6N2Y0ohzlHVahaovm7ojqPUl4oIZ6n42RnN+B5ZBTUdGBOykP13t7A/5DmvTuRmy0vQ74trUKW\n", + "M14L3EjX2sReyL9LPXA7EJ58rTbNuQYjswhMRG4srUeaCWPAN4HFwKYghhHBeyqBPwTb65BpcsYi\n", + "fVqHIEswNwALevg7j0Ca5bYAvwjOmSi0ZwD/Ds6xMThfTfDazcAU4C/Beb4WbP8jcl22AI8gn9GE\n", + "o5EJROuRa7hP6LWPIiN16oDHSX5+M52nv6l1HYBHal0H4FopHa5D8FLOTVs2OWyhz49exFsBrECG\n", + "M5cCxyP9R5cChyIj3K5EJsaspHMiSbz3guC9JyLT/l9K9w6la9PWBchUNRODc/0amVMN4CzgvuD8\n", + "MSR5JdYvyLZ5ajRSqJ8QxHph8Lcl3jsDWYagPNj3EWRanoRldG1ymo8kxvJg3/+FXltLclh6TRAz\n", + "wc/1yDD2GHBacOzExKPpzpOq2Ju2lHrPMLbEo+wjKRbFlkjmId+Ywx4lmUhakISRYEgmgXnA6pT3\n", + "Pk7v+kheo3MBOgFJSqVIs2L4m3vYw2TXR3IakqjCVpI5CX0MqWkl9FTAD0f6zRIJbgXwBWBY8NwE\n", + "P39F10T7BlKzyuY8UPz/4YzrADxiXAfg2nA2d+QjkWTbtNVvxSCWr0cvTj+RrslgJcljbSS5uFg2\n", + "711B7+KYCvwZae6pQxJLO9KEdTPwT6TpbDUyei7ct5bNB3AiXRNmOJmNC46/CrlX6GZkgtBMSoAf\n", + "IU1xW5EEYJHaDEjt7Gik87yWZLPXjsBXSf6ddcCkID6lBqK8dJIM+ETi2Fpgh5RtU0gWzt0V0une\n", + "u2PoPbUZ3pfumO8AH0b6RRKP6uAc7ci3+NnIfGgfRWoYPcUXtobOszvHUp5fgYzMS8yW8Bk6fzZT\n", + "z3MqcCzSHFYDTKNzn8uzSK1mDHAP0v+T+Dt/mPJ3DkH6hHL5e4pZresAPFLrOgDXqtmel04STSRu\n", + "PYEUoF9CvuUfh7TfQ881i/8ihfz5SBv/CaH3dmc98m1/WGjbr5HCfErwfAxSUINU/+cgzVwNSN9G\n", + "4sO3Hunf6MnfkER0PPJ3nk/nudWGICMB65HkeFGamGek7N+CzMQ7mM4zR5cjiaYmiLMhFO/vgLOB\n", + "/ZHrOxj4CMmF2rL9e5TqF0ZQl24y3gGvGO8j2QfpKG5ARm3dDXwX6SN5J2Vfk7JtHzqP2rqNnvtI\n", + "QEaAbUIK4sSorS8j/QX1SJPR5cG+nwi2bwPWIfM0Jb6AHAC8GRwnMX9TJh8K9k2M2gp31O+G1CIa\n", + "gr/nKyl/57FIs11d8NpgpKZRjzRrfQZJFtORRPKPIKatyKwMX0yJ4+ngWGuQ2kgikaSeJx2fP0vZ\n", + "MK4D8IhxHYBre/BCfZSd7fMyPA5BvgWPjCKILBRjIkn1FHB6Ho5j8nCM/sLk8VjF9FlKx7gOwCPG\n", + "dQCuzeX5LVEmkjjp7y6Ph177M/JN0aViTCTzkFpBGZJAGpHOZ+Unnz9LSuVkH56pi3LU1tHA60jb\n", + "887B41Tkpq+TkPb5uciIHpWbWSRvkPsycj1T12vJ1beRZqLUx9/6eNzuHJLhnPUFPKdSqg9KiOfn\n", + "1vYsPQt8MM32DyJTpoCM5lkeVUAZFGONpFCM6wA8YvJ4rGL/LBnXAXjEuA7AtQN4YlOUNZLZdL1n\n", + "AaSzcnbw+ytkv8qhUkopx6KukTyPzH80KLStErlxLHEH8iHICBqXtEaiCk0/S6rfOJSH10Y5++85\n", + "wF+RGsjLyHDR3ZFO9mOCfaYD1/clGKWUUtEZw8bmqM85BJnA72fB4wskx9/7QmskScZ1AB4xeTxW\n", + "sX+WjOsAPGJcB+Dax/jT61HWSEBuSEtdIlcppVSR6qA001x+BRHuG5kE/ABZMXFelEFkob/XSPK1\n", + "1O45yBDjepLrjmRSS+YZfqci9xENpKl2+stnSSmO4d4Xohi1NQuZCbYJuddhNnLn9VeQOYv+jcyf\n", + "pKLxKLBLN69n84EoB65BJjwchty/0h1dq0CpfiomSaTPekokVyMd7Mcinex/R6YUrwkevwG+kY9A\n", + "VI+ybYZMTPZoMrw+Hhlx93pfAyoixnUAHjGuA/CIcR1Af9FTIjkAmYn1r8C5yNTf15OcGuWXwK6F\n", + "DHAAKNRSu+nMJJlAtgAPBr8fCDwTbHsaeH+G95ciXy42AkuQmXOVUkUqXzWSnsSRxY0SGpBhvgnj\n", + "ydPCKHnSiz6SvC2QWCxL7e5I536NkUjz1qnBtk8gM+cm+k7Cs/SejSSiHYLXH0aGgGsfiVJF6Dj+\n", + "/KwPKyTqf6q+OQBJAr8gOfHl06HX48D3kYSSOt77AKS569rgvXcjtYqepK5z8hFkevdbgvPdjnTo\n", + "H0tXJyNDv1cjyeeKNMdTShWPvJTh2bS734x8S44h34p/i3S+W3puSikCMZcFYSGX2jVktwLcRLqu\n", + "e7KC9MvPTqBz01rq+3xl0NXwEgx6LRIMA/xaRNXZ/n9IZ/tm4F3kW+uq4PfNwWu/z0cgA1Qhl9rN\n", + "1urgfanHSTe32lqSqyiS8rtSSvULxXYfSTny7T+81G6ij8TQ+ds/pO8jCS+1m00fyVTS95F8Mojh\n", + "FORLQmKxstQ+kldJ9pE8hN5HolTROp67n/Khj0T1TRuSAD5HssP7r0hCyPSPm9jWGrx3PlJDPBnp\n", + "J8lG+LibkSUAvoosv/u14PnmNO/7HTL8+0VkaYG7M8SolCoO+v83jWKrkaSjS+3mn8njsYrps5SO\n", + "cR2AR4zrAFw7nruf0BpJ/5C61O7uwP1OI1JKDQglxIv9i1FBFGON5ExgHXKPzgvAUXk4pouldgcK\n", + "nz9LSuXkRP74WD5qJP1NMSYSVVz0s6T6jZO481Ft2lLdMa4D8IhxHYBHjOsAPGJcB+CBSO4jUUop\n", + "1X9pDTsNbdpShaafJdVvfJw7a4u1aetcYBkyzcqzwME97D8HeATYjtxV/71enLOO5H0Z+tBHXx49\n", + "rd+iVBGxPk26m7VTkBvpPocsmrUAGU00OcP+w5ARTbcDuyEz3NYjC2ulY/MZbJEzrgPwiHEdgEeM\n", + "6wA8YlwH4NrHueMhinDU1lN0Xff9LWQW2XTOQdbICC/1+x2kZpJOUV2MArvQdQAe0WuRpNciacBf\n", + "i5O5/cF8JJIom7YqkEWcHkjZ/gCysFI670eWl21J2X8iXScaVJ0Ndx2AR/RaJOm1SNJrkacv31Em\n", + "ktHI2hvrU7ZvQO7sTmd8mv3Xh15TSinVS1FNI++aNlX13lTXAXhkqusAPDLVdQAemeo6ANdieeps\n", + "j3JRpwqgEVnKNTxL7XVIR/phad7ze2AUMhttwn5IX8s0ZBr1sMXAjDzFq5RSA8USYCfXQWTrSdJ3\n", + "tv8ww/5nA1vp3Nn+bbqu06GUUmqAOBnpOP8csCuy3ng9yeG/VwIPhvYfhqzKdxswG1l/Yyvw5Yji\n", + "VUop5aFzkBsSm4Fn6HxD4kJgacr+uyM3JDYhy7/25oZEpZRSSimllOo7F9Or+CqXa2GAe4E1yICH\n", + "F4HPFji+qOT6mUjYmeQ6Lf1Fb67FhcAbSAvBGqR5uT/I9VocjfTh1gMbgXuQz0ixmwfch5R/cbJb\n", + "fbU/l5sFn16lmOR6Lb4FXIrc4DkVGcTQBnyy0IEWWK7XIaECeA74K/KZ6A96cy1+CrwJHIN8LvYE\n", + "PlzQKKOR67XYKdj/R8B05DrcD7xd8EgL7yjgcqT8awRO62H//lxuAoWfXqWY5Hot0rkDuCtvEbnR\n", + "2+vwM+BG5NtZf6mR5HotZiGF56xCBuVIrtfiJKCdzrdDHIZ8gx+Z9+jcaaDnRNKrctP3GxITdHqV\n", + "pN5ci3RqgM35CsqB3l6HjwSP84j2PqpC6s21OA4Z2HJ08HMZsAgYU5gQI9Oba/E4sA1Z9roUGArM\n", + "B56muP+P9Eavys1iSSQ6vUpSb65Fqo8ChwO/zWNcUevNdZiI/M2nIu2//UVvrsV0pGA4GfmW+hlg\n", + "F+AvFHeC7c21WIsk1MuRvqItyO0GxxQoRp/1qtwslkTSGzq9SnoHAbcg38ifdRxL1G4GfoUMOx/o\n", + "SpDmi88AjwWPzwD7A/s6jMuF6Ujn+kLkbzdIM9CdFHdS7Y1elZvFkkg2AR3AuJTt45BvE+mso2sG\n", + "HRd6rVj15lokHAz8HRmFkdqGXGx6cx0OA76PDDRoA24ABge/f74wYUaiN9diLdIvsDi0bXFwnCn5\n", + "DjBCvbkWZyGzZXwDGdH4KPBp4FCkqWcg6VW5WSyJpBUZZXNkyvYjgCcyvOe/wCF07jQ6ArmpMXWO\n", + "rmLSm2sBMgzw70hBuqAwoUWqN9dhd2RETuJxMTI8dE+Ke+BBb67FY0AZ8m08YTrSLDTQ/n/EkI71\n", + "sMTzYikj86W/lpvv0elVknK9FgYZ+ncV8u1ifPAo9o7VXK9Dqvn0n1FbuV6LGNK0WQvMBfZC7h3o\n", + "7stIscj1WhyM1GK+h9w7sjcy/Hc5UBVJxIUzGPn3nYuUAd8Lfh+I5eZ7dHqVpFyuxULkP0o85ZF6\n", + "vYpRrp+JsPn0n/tIIPdrMR7pB6hHOlRvpvi/XCTkei1OQhJrA3It7kEGHxQ7Q/L/e7gMuCl4faCV\n", + "m0oppZRSSimllFJKKaWUUkoppZRSSimllFJKKaWUUkqpfq4WN9O6zMf/O98XITP2dmcqchPa3jkc\n", + "dxn9aDEjpVT/Nga4nuRdyOuQKRs+GNpnODLVQ9Tm438iGYpMa5FQC/wiZZ8SYCwyj1a2RtF5epA4\n", + "Ml2GUl2UuQ5ADXh3A5XAGcjss+OQWVfDK9NtcRBXscgm0cWR9Thy8W6abQNtSnWlVBEYjhRyh/ew\n", + "Xy2dv2WPA+5DFqdahizM9Aoys3FCHFnx7o/I6ndLkAWtwn4EvBE6zlV0nvV0Pj0X1Gchy7g2ARuR\n", + "yf7C3/w/C7wWvP4mcCGdC+Rs4rwYmUCwGZlQ7/eh1xaRbNpaRNf51KbQuWmrBJky/Usp55gZ7DM3\n", + "eL6cZNPWcrrO0bZj8Ps+Kcc5M7gO+iVVKRWJMmTCwGvpXICnepjOfST3A/8D3odMAf9gcJyLQ/vE\n", + "kQLzU8j06FcgM8JODu3zXWS9iSnAUcg02ZeGXp9P94lkX2Qtk08Gx90DuIBkIjkTWIM0Ce2IrEy5\n", + "FvhiDnGeiMy+ehQwCSm4zw29fyGSVEGauB5H1lkZGzxK6NpHchUyXXjYD5BknBDuIxkdvP+M4Jij\n", + "gu33A9elHOe/wDUopVSETkCaUZqQKcx/gqzSFxZOJLOQQi28zyRkkabURPLD0PNSZBrtT3UTy9nA\n", + "26Hn8+k+kZyANLsNyfD6O3StXVwIvJpDnF9Bak2ZvuEvonNne2rSha6JZI/geXgtkreBb4aep3a2\n", + "p+sjORFZ0zzxJWDXYL/dMsSq+qmBtmiL8s+fkLXUjwH+ARwIPAl8K8P+uyCFVXiZ4FXIN/9UL4V+\n", + "70CaXMaGtp2ELPC0FkkYP6VzjaUnDyC1mGXAH5AmtkRSGYMkuN8Gx048rqRzAd5TnHcifUjLkJrG\n", + "SUBFDjGm8xLwMskk974gpltyPM59yEJSiQRzBvAU0pSnBhBNJMoHLUjz1GXImvI3ApfQ93b2tpTn\n", + "luRn/gBk8Z5/IE1Oc5GmrlwK6W3It/yTkdrHt5Daw4TQec6i86qMs4NHtnGuQmphZyHNd9cgKwBW\n", + "5xBnOn8gmUhORZaXXZnjMdqA/0MSSCmy5vuNfYxLFSFNJMpHryNJpDLNa28gn9t9Q9smIbWaXByE\n", + "LNrzQ6RgXoI0AeWqA2lO+jbSZDQY+AiyONIaYCekczr1kYsWZJnkrwD7IYnowAz7tpJdAr4tiO19\n", + "SCL8Qw/7t5F++PANwGFIv88Q4PYszq36GR1ZoVwahYxWuhFpamlAEsTXkRrKtmC/GMmRTm8C/wR+\n", + "jayC14L0qzQh3+S7Ex4t9SawA9IX8STwIeATOcb/EaQw/g/SV3AYcl/H68Hr30dGm21Baj7lSA1m\n", + "IjJiLJs45yMF+NPI9TgFSRZvd30bICOs9kc69xtJP4wXpKbzCPAbpJP+j93EkzjuB5GaSwtQF2x/\n", + "C2ke/DGSnLale7NSShVKBVIjeBopiBuRgulqZGhwQmoHcmL4bxNSwJ2O3INyUWifdJ3DqR3IVyD3\n", + "VzQAdyGd7R2h1+fT/VK8BwH/BjYhQ4hfCmIJ+wRS42kK/sb/IDWAbOM8DhmEUIcU0k8BR4f2DY/a\n", + "Allz/AnkWnaQHP7bQdc72z8bnP+uNH9b6rX6KPJv00rXGtVpwXEORimlitRo5Fvy8a4DGaC+gTQ5\n", + "KqVU0TgM+aY+Hek0fwhpqqnq7k0q7wYj/TWrkWZGpZQqGkcizUjbkKape+k6pFYV3iLkbvs70IE7\n", + "SimllFJKKaWUUkoppZRSSimllFJKKaWUUkopVQz+H82vTWAqrap6AAAAAElFTkSuQmCC\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "report.roc().plot()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Metric" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "collapsed": false + }, + "outputs": [ + { + "data": { + "image/png": [ + "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEZCAYAAAC99aPhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\n", + "AAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd8HMXd+PHPSS6yLfduwJYLBmwIBlNC9QAPkAAJLQQS\n", + "AiEFAwkJHUJJMCGEEh5K8qMkBHAgCYY8IYFAQoDgoxlTYpqNKTbuvUqyJdkq8/tjdn2r1Um6k/Zu\n", + "Zu++79frbO3e7uz3Rqudm5ndGRBCCCGEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBAieouBpjSv\n", + "Z7z3E8A0YAVQA8wEJuQ7SCGEEO4aCAwJvCYBjcBZ3vtXAVXAycBE4HFMoVKe90iFEELEwrXARqA7\n", + "pjayCrg68H4ZpmCZmv/QhBBCuC4BfA782lseg2nmmhza7hlgev7CEkIIkY0Si8c+GqgAHvCWh3n/\n", + "rwlttzbwnhBCCMfYLEjOBd4CPsxgW53jWIQQQnRQF0vHHQJ8FfhBYN1q7/+hwPLA+qGB94JWACNy\n", + "Ep0QQhSuhcA420FE4UqgEugZWJcAVtKys70SU3sJk1pKyjTbAThkmu0AHDLNdgAOmWY7AIdEfu20\n", + "USNJAN8HZmCeFfFp4C7gGuBj4DPgOqAa+HOeY4ybCtsBOKTCdgAOqbAdgEMqbAdQyGwUJAoYC3wz\n", + "zXu3AT2Ae4D+wGzgGGBrvoITQghRPKRpK0XZDsAhynYADlG2A3CIsh2AQ+TaGSCZIYQQ2Yv82mnz\n", + "9t9c2YjJKHnJq7OvjcSbsh2AQ5TtAISbdJbrC5myHYBDVIRpxf1cUrYDcIiyHYBD4n5eR0oKEpFr\n", + "ci6JQhT5eV2ITVtCCCHySAqSwqBsB+AQZTsAhyjbAThE2Q6gkElB4rbDMA9ntmY6cGN+Qum0JPA9\n", + "20EIIaInBYnbXgV2b+N9/+6iZBvbKGBZRPEk6Xhh4MeaicXAkR08TjKLbTtznDhI2g7AIUnbARQy\n", + "KUjclemoA4mcRtFcvjqfNfn5XPk6jhDCUXG9a2tf4F3MzI9PYKYTvhFTc1iOGdByFfAHWtYm9gHm\n", + "ePvOAB4L7JtOL6AWM51xtbffMMzF8yfAAmC9F0N/b58y4I/e+k2Yof6HADcBDV561aQmJGvN0Zhm\n", + "uc3Ab2hemxkLvOQdY513vL7ee4968dZ4x7ncW/8XL182Ay8DEwLHOg6Y532+dcBlgfdOAN7zPsvr\n", + "wF7tHCfI9XOpPcp2AA5RtgNwSNzP60h1qCDRoKN6dSDmbsAS4EdAKWZu+m3Az4EpQD1wM9AVc0FX\n", + "pAoSf9+LvH1PBbZ7+6o2jjmFlk1bFwGzMMPwdwXuJzUw5nnA097xE5jCq7f33kzguxl8zkGYi/op\n", + "XqwXe5/N33cscJR37EGYguHOwP6LaNnkdA6mYOzqbftu4L1VwCHez8d7MeP9vwbY3/ssZ3tpd23j\n", + "OEFx/4NTtgNwiLIdgEPifl5HKo4FyeE0n2sFTD+IX5BswxQYPkWqEDgcMwdL0Ovevm0JpuH7iOYX\n", + "0OGYQqkU+A7Nv7kHzSSzPpKzMQVV0DJaL4ROwtS0fO1d4PthpmX2C7glwFSgT2i7+2iZPx9jbmLI\n", + "5DjyBycKkTxH0lkJSET16sDhR9CyMFhGKq11mAt6pvsuoWNxVAB/wzT3bMIULA2YJqxHgX9jms5W\n", + "ALfSvL8mk5NwBC0LzGBhNtRLfzlmvplHgYFtpFcC3IJpiqvEFAAaU5sBUzs7DtN5ngS+6K0fhWnm\n", + "2hR47YxMiCZEpIquILFsFbBTaN1IUhfnti7S6fYd5e2j2tgvXZpLgS9h+kX8V0/vGA2Yb/ETgYMx\n", + "fQxnZxBf0Epgl8ByIrT8S0z/xJ6YvpGzaH4uho9zJmZGzaO87Ud7afqF6DuYWs1gYC6m78n/nDeF\n", + "Pmc5pk8om88TV8p2AA5RtgMoZFKQ5NcszAX0Qsy3/BMx7ffQfs3iDcxF/seYNv5TAvu2ZQ3m236w\n", + "2ed+zMV8pLc8GHOhBvMHtxemmasa07fRGEhrbAbHfBZTEJ2M+Zw/xnTy+8oxc8xUYQrHK9LEPDa0\n", + "/TbMIIq9vNh9XTEFTV9Sned+vA8A5wMHYPK3F6YPpTzLzyOEKFBxvWtrMqajuBrzzfmvmJkgp2C+\n", + "QQep0LrJtLxrq70+EoAHMXdIbSR119YlmP6CKkyT0S+8bc/w1m8BVmNmrfS/cHwR+MRL5652jnms\n", + "t61/11awo34CphZR7X2eS0Of86uYZrtN3nu9gL97sS7C1GAagTGYguRfXkyVwJuYmlQwjre8tFZi\n", + "aiN+QRI+Tpjr55IQHSHndUBcC5KwN4Fv2w5CpBW3c0mITMh5HRDXguRwTK2gC6YA2YrpfO4M1cn9\n", + "C4mKMC3Xz6X2KNsBOETZDsAhctdWAdiN1ANylwBfw7TVd8aZmGai8OvZTqbblsNaOWZVDo8phBCR\n", + "imuNRMSHnEuiEEmNRAghhFtsFCTDMeNIrcWM2zQP028QNA3zMFwN5m6fCYi2KNsBOETZDsAhynYA\n", + "DlG2Ayhk+S5I+mGG39CYJ5F3xzxTsTawzVWYWzEvxDwnsRZ4gdQtm0IIIYrYLzFjS7UmgXm6+urA\n", + "ujJMB+7U0LbSRyJyTc4lUYhi30dyEubhsMcxdyq9C/ww8P5ozK2wzwfW1QGv0PwhMyGEQEMPDd/Q\n", + "ZrgdUSTqMP0iNwF7Y4YGryZVmByMGdV159B+DwHPhdYVQ40k06l2VTvpXIApuKtIzTvSmiStj/Bb\n", + "gfn9uHyThoowrbifS8p2ALmm4Y7AiNzjW9nqWrjnd/mNzGmRn9eZzsIXlRJMjeRab/l9YFdMQXJP\n", + "O/um+/DTMSO+ghmK473Ae8r7Pxnz5d3beH8oqaFFWtv/deB/MXm8CPP8Slvb+1Pipns/OF5WpvHn\n", + "eznq+KJOL5/LkxyLJ/LlSvpcMp1z6Ma/Gc0nM0kNbOptr18DfmEG1S75MzQ5FX+elhXmSzukrpex\n", + "thgIfzM4CzOuE5ixk5owY0oFPQs8HFpX6DWSTAr5hzE1krbsgsnT0gyP29bkVRW4XyOJUqGcSwVJ\n", + "Q9fgFEEz+LrWcLGGHoGtBga2yfcXZ1fFvo/kdVLfsH3jSZWSizADBR4TeL8MOJSWEyXFVaZT7T5I\n", + "+1PtlrVzrPHAfO/nzcCL3s8HA297694CDmpl/1Lgdsw8KQsxI+cK4QBdfgl3/CC45gweJ4G+M4Gu\n", + "6UX1L7T5+zghsElXREHYD1PHvAYYB5yGuZhdENjmSm/dyZgONH8CpF6htDpYI4lsgsS4TLU7iua1\n", + "iAGY5q0zvXVnYEbO9ftOgjWS8zEF0U7e+zMxo+66XCNREaYV9xqJsh1Ae3TmNeXgXr0y/Rvtw+af\n", + "m59natC920+7KMT9vAbM8yPvYTrdP8Y8LxJ2PWbI71pafyAxjgVJrqbaVW0cs4LmBclZwOzQNrNI\n", + "jUAcLEheovlt10fjftOWijCtuP/BKdsBhGnoq+H3Gu4K/CFN09CveZNU6xQv/TTN3+Pf2v5bnalB\n", + "D8j154uJ2He2A/zTe7XlBu+VA4mOTE0blVxNtZvMMoaloXVLSD/97HCaN62F93NR0nYADknaDsCn\n", + "zReYcuDe8HuzOOj6Xfns+nK2oKkrT5gRsQFdApRCoj64/Qp2Cj9TBnDZcFaO38iACdsCLb5l1DbU\n", + "0aOLV6Z2oPYjMuHyN8tClKupdrOxwtsvnE64kPKPOTKwPDLNNkK0SUN5PV0eAe79NT+ijFqm8lue\n", + "4Xj6splDmMUQ1tGTWo7jWa+Q0CWYZtTtoAOjWuiyzxgffjwAYNEqdppYR4+B2+m644YeU4gYPagp\n", + "1XCNhiNz80lFHMXxrq2umG//wal2/T4SRfNv/5C+jyQ41W4mfSQVpO8j+YYXw+mYPhK/2h/uI5lH\n", + "qo/kP0jTVpyo/B9SHw/6Z/4dUhpK1jBYl1OVUXNxN+rWApTQcFxg/fVe2iXQ9KC//n94fmkJDdtL\n", + "aFgIullLwzk89Fq4aetpTvhmYMUvdcvn1YpF7O/aKnb1mALge6Q6vJ/BFAia9L9gf912b99zgA3A\n", + "1zHT9GYimO5GzJ0sl2Gm373cW96YZr8HgH9jnvd5xzte3C+uIjK6FPQtoC8yF3ndHXM+3wB8Cnr/\n", + "I3hpzlOcyBYy6+cez6eLAfZlzi2B45zn9Uk2QmLHrekHM+uIJkoHNVG6NySanZcfstfqcNpjWXhW\n", + "YPFq4JbwNqL4xLFGko5MteuuuJ1LeaZ/0YmbVa4I/Py4//OtXKE1PNDe/j/g/7X5uymn6tTwPp9T\n", + "oVcyTN/HebqKcq3h6XzllGPkvA6Ia0GSi6l2RW64fi5ZppdnWXi8Cvow0D29/QeCPhZ04gpu3bFd\n", + "Ld11D7a2mdbf+erf2oktAfpo0I+k2/9A3tAa/pH7PHKSnNcBcS1IzsU8dFmNuQ36yxGk+QD5n2rX\n", + "VSrCtFw/l9qjcpe0HgG6Kd1F+qfcoKdyf7oCoNWpIE7j8Znh7RM06kc5U4PWv+AaXUt3fS/n62u5\n", + "UW+ja8aDNPZki3/7b7OXLs6/D4j/eR2puBYkuaBsB+AQFWFacT+XVO6S1qeB1kfyop7JFP0hE/VG\n", + "+jW7Ul/LjcHFUW2l1pvK/l3YPi94oe9ObVvVm/ZGdQjGmrYg2YN5m0GP6WxOxFDcz+tISUEick3O\n", + "pRZ0AvTcUBNR+HWnhpLtdLn0Au5Z+T0eaG3stnTpvxEoSOrTpL1Jw8AsY26rmWx9dmkVBDmvA6Qg\n", + "Ebkm51ILemLwQnw5t4WvzOkebM0m/R6g54Oe/Qjfqgiku18n0rwCml6/gHvOAK37smlB85CLTjF+\n", + "5lZJQZKibAfgEBVhWnE/l1TUCXan9nL/Avx7vutfiYdqM2S9y5T/g4ajL05NYxJ+dqsYxP28jlRr\n", + "mbHRe09e8ursK92zNXGiok6wG3UatL6Ra/1C5LWoj5Ejyv9Bw1Gb6aO7sN37CLrY7prUtgNwiWSG\n", + "EHmkYYh34dXvsrdfkMTuoWYNR2rQ+/OmX5DcajumPIv82hm7k0AIkV8aemvQc5m4xl83gY/uA8oT\n", + "ZsicuGkA2Jc5APSgJjyGnSgiUiNJUbYDcIiyHYBDVBSJ/Jujl9HibqfYUf4PGnpq+OwRvqVB6+Gs\n", + "KLbnSaRGIoTIn6ncf8yxPN9scMNv8OcZtuKJQgJqgN22UD4XYBP9D7AckrAojt+KhIiNVzlkzGDW\n", + "6HBt5BieK4iZBmcy5ZnA5zrEdjx5JNfOAMkMIXJEw8T57Laj8PgJv1wK+kHQQ2zHFhUNtwWGcvmX\n", + "7XjySK6dAZIZKcp2AA5RtgNwiOrIThqO3UqPHYXIGBYs0alZPONKhVdoOO0Tdg3WtgqippUB6SMR\n", + "QuSOhmGzOfC5XtTsWLeYigWJwvzi9vp4PmsaxDp/OYvxu0ShKMQTWwirNLze8g4tPdh2XLmi4cVh\n", + "rPQ/53Db8eSJ1EiEELmhYZcbue7g0OqekFiXdofC8HEX81gJmDmCRAxMwzzAFHytTLPNCswtejOB\n", + "Ca2kJTWSFGU7AIco2wE4RGW64XoGHLArn+yohfSmMvx3GXcq3UoNv6rgc/9zj85zTLYURI3kY8wM\n", + "gf5rr8B7VwGXAhcC+wNrgReAVifEEUJ0RGoU3z5sfm4QG978jPE73h3E+hMsBpdP1X6N5HBenv8G\n", + "BxbTbcCxNQ34sJX3EsAq4OrAujKgCpiaZnupkQjRIfqWNP0gwVeW833El4bTd+ejHZ/9x9wVxyFf\n", + "slUQNZIxmKarz4HHAL86ORozd/nzgW3rgFeAcLutEKJDdAJT829hH+asgUQCEhvyHJRNiz9mjx0L\n", + "qxkW99ucrch3QTIb+DZwLGbu8mHALGCA9zPAmtA+awPvifSU7QAcomwH4BAVXnEmf/yH//NLHMEH\n", + "7MVDfAdNYtUcJn8hr9Hll2pl/ZbgQiOlaBib+3AKS77vUngu8PNc4A1gEaZwebON/Vqrik0HFns/\n", + "bwbeA5LesvL+l+XiWqad94tpeVJw+RbG3fITvnUgwBgeqD+Kl49pQh+1F3PXJeADzI0tax2KP8rl\n", + "Sa28/2pw9QLGARwC7GI53iiXFXCOt7yYAvUScA+maasJmBx6/1ng4TT7SR+JEBl6li+/HewH+ZyK\n", + "UbZjckcqX/bkA63T98kWkoK7dpZhOtiv85ZX0rKzvRLTDBZWcJkhRNQWMnr8U3xldfBiWUr9FNtx\n", + "ueQknjz2OJ5ZCFrvwTyt4Ue2Y8qx2F87bwcOx9Q+DgSewTRJ+dXIK73lk4E9gRnAcqBXmrRinxkR\n", + "UrYDcIiyHYArynmyMnxHVk+2/K/tuCxRbb+tdwOtx/Ox1nBZXiKyJ/Z3be2EuVPrY+CvQC3wRWCZ\n", + "9/5twJ2Ypq63MXdxHQNszXOcQsSYLhvC6ue30L9PcO1ZPPI/NZQX+kWyo5rMPyUA3e2GIvJJaiRC\n", + "pHEor0wL1kIu4J4HvNt+Rav0GNB6NAu1hhtsR5Njcu0MkMwQIo0KPq/yHq7TWka0zZCuAK1HsUhr\n", + "uMV2NDkW+6YtkRvKdgAOUbYDsOm3nDtkMaN7AzzLhr8nzEO9ov3zotH8UwrStJU1KUiEKCC92Pof\n", + "/+eF/Oo3NmOJGekjKVLStCWER0OvVzlkk98vMpwVK2zHFC96GGg9lFVaw4O2o8kxadoSQrS0gQGn\n", + "n8KT/fzlBrpcbzOeGGoCadrqKClICoOyHYBDlO0AbFjBTlPWMcRf3GkdQ39PkeZFK1Q77zeCNG11\n", + "lBQkQsTcJObctDcfnA0wkPWrIVFok1Llg9RIipT0kYii15Mtfwo+MzKC5T+0HVM86T7ezJBaw79t\n", + "R5Njcu0MkMwQRW04K670C5CdWaqv5/qjbMcUX7rcG0JG69QIuoVKrp0BkhkpynYADlG2A8iHwazZ\n", + "3S9EDuUVvY2u302zmcp3XA5Tbb+te/j5uZohWsPdeYnKDrl2BkhmpCjbAThE2Q4g93Tiq/x9x62+\n", + "NZRd18qGKp9ROU61/bbu7udnGTV+W2Ghjgog184AyQxRlBQv3elf9M7n3vlabpqJgC4J9jV5P1Rp\n", + "uEObwWYLiVw7AyQzRFE6gv+8B1rvyzsN+/JOvmc5LWAtChL/9Y/2940VeSBRpKVsB+AQZTuAXCuj\n", + "bhTAnsy9Zw77NbSxqcpPRLGgMtjm162s3zXCOAqSFCRCxIiGk+cxsR/Aaxz6lu14CszdAF3Zviy0\n", + "/ikLsYg8kaYtUVQ0HPQ3Tgy2ugyyHVNh0SO9fF2qYXAgo39qO7KIybUzQDJDFBGd+A4PrvCvbV3Z\n", + "9rztiAqP3snL3xUAGm6UgiQz0rRVGJTtAByibAeQC7/hwmse5rsjulBPHyofq6fbWRnspnIdV4yo\n", + "DLbx+5vkBoYsSUEiRAwsYNz3APbh3RVV9PsmJNbYjqkA+QVJqdUoRF5J05YoCm8zecdT14NZ85zt\n", + "eAqX7hfofxoXaNr6me3IIiZNW0IUm1kcvMj/eSMDfmEzlgIXvJX6kyYS8mU1Bq7GDN0cng50GrAC\n", + "qAFmAhNa2V9+ySnKdgAOUbYDiNKF/HpcfzZo7wHEf2W5u8pFTDGl2t+k+dPtoPWvuExqJBmwVSP5\n", + "InAu8AHNP9RVwKXAhcD+wFrgBaA83wEK4YKe1PxsEwPYieV6DpNPtB1PYUs0AZODa67gdu7ionGW\n", + "AhJt6AssAKZgahz+06QJYBWmpuIrA6qAqWnSkRqJKGgfM34//5vxFGbOtx1P8WheK9mPtz6wHVHE\n", + "CqJG8jvgL8DLmMLDNxoYCgTvj68DXgEOzlt0QjhBD9udT972lzYy4Aab0RSZw4MLvamusxVIXOS7\n", + "IDkXGAP4w14HS8Zh3v/h2xrXBt4T6SnbAThE2Q6g83TZzix70186hb/e8CF7z+hAQiq6mGJPZb5p\n", + "4lVIJL7KU7MA6ukqz5W0I58ZtBtwE3Ao0OitS9C8VtKa1qpi04HF3s+bgfdIzW6mvP9lubiWaed9\n", + "x5f1a32ofGs5C0cOZhbPc/OV+/D+rzqY3qT8x+/s8qRs9x/Eb6rhRN7igIlQdiTUNVmMvzPLCjjH\n", + "W15MzJ2DuUurPvBqwhQq24Hx3vLk0H7PAg+nSU/6SEQB0kf5bfP/xyl/sx1NMfsXx94T6Cu5xXY8\n", + "EYr1tbMv5lZe/zUReAv4o7ecAFbSsrO9EtMkFhbrzBCiJT2wJ1seAq1P5S9aQ0/bERWzhYy+tXnH\n", + "e8EopM8CmGpX8DmSKzFNVCcDewIzgOVArzT7FlxmdIKyHYBDlO0AOkZPKaW+3r9oXcSdKyNIVEWQ\n", + "RqFQ2e6g4QYpSDLTXmf7SMwFPehIzG27b9G89tARmuYf6jbgTuAe4G3MXVzHAFs7eRwhnDaKxU83\n", + "0qXLPszhMc7gF1x3he2YBLzA//g/vmIzjrj7PyDYNjgS2ALMBZ7G9HP82EJcIDUSUQA0DPkVl/0K\n", + "tC6hQW+lx3zd+mgOIo803PA+e/k1krm244lQ3q+dS2h+T/XVmIcJu3rLlwNz8h2URwoSEWO6/Epu\n", + "eWMiH+5oOvkis1bYjkqkaLhhFUP9389a2/FEKO/XzlpMLcT3b+D2wPJumD4NG6QgSVG2A3CIsh1A\n", + "63TZANZfMoLln3Rhe1Ow/f0YnvvnVO7vF/EBVcTpxZnKdgcN0+opDT7lXhF9WFbk/dq5CtjH+zkB\n", + "bAS+Fnh/PKapywYpSFKU7QAcomwHEPQp48Y+zQk3H8mLCxI0hofeaDyI118APSBHh1c5SjeOVLY7\n", + "aJimQfdga633O3sEdCHMVZL3a+dTwKOYBxe/gXneI3jSHw/YGgNIChLhrP/lkt1P57FZPdnSrPAA\n", + "rX/CL5fNZ7cfaehhO07ROr8g+TLPzgr8/gqh0z3v1869gfWYcfqbgPBcCI8C9+U7KI8UJMI5e/LB\n", + "90toaAoXHhP58L2ruPmHW+kx3HaMIjN+QXIB9zxbYLcBW/kMg4GTMEO/h52AGWzRhkL4hUZF2Q7A\n", + "IcrOYXX5Hsx7M3jBGcLq2qP59x2gu7a/f04oS8d1kcp2B78gWcfAm6QgKVySGSnKdgAOUfk7lO51\n", + "CK/eNok5C8I1kNN57Ec6s3HkcklZPr5LVLY7+AWJhmmgh3m/26roQ8u7yK+d7Q3aeGqGB30yglhE\n", + "xyVtB+CQZL4OdBQvPvof/ufk4LrruHHJ13nigC8wd+3j+QqkdUnbATgk2Yl9E5gZW0UHNWX4skFq\n", + "JCLvNCRO4snvjePTT/zaRwWf1/6FU8/TphlYFAgN13s1khtA95EaSevaq5Gswjy9Ph0zAu+SqAMQ\n", + "kVDIt0+fIqK86M+G3nvz/peGsfrgUhpPXcmI4SMZ12VZ4NGqA5nddBivHn0af30timNGTCHnhU8R\n", + "TV7Ybq50UnsFyUjgy8D3gI+BV4EHgb9hbgUWIva8voyh93H+F99lnyMWMG7/eUzcey0DeiY5osX2\n", + "ZdRyKn/96Dx++/vDeO0PCfN8lShs0gLShmxK1yHA2cB3vZ//jBkixVaBopFvByJLb3BgxQLGfWsL\n", + "5fsvpmLcOgYPX8fgvu8xqWRZs0EcoDt1jGZRYwlNWyfz33krGfGHBYx7dQkVn0KiwdJHEHmi4Xpg\n", + "GvDzBPp2oArYAoneVgPrvMivndnMkLgWMzzK08BvgQsxmSzfxoQztBkH7gvAyBWMGPsSRx48h333\n", + "XsjYPsvYpd9h7NmlgfR345ZT3bQnc9cOZ9Wyw3j1pdN5fPoIVn2SAP1RXj+FcJh8eU0j04KkF3A6\n", + "pjayD+YurSORQsQVigJuC9fQbRXD+iRRIzYwsCKB3nML5cO2UD64hp4Dt9KrPzCqkr5dD2VBn82c\n", + "XLKUkVTTJ216g1m7rS+Vy8qoW9iFhrkrGfHeWoZ+sIXe82ZzcCOYtttL8/gZc0RRwOdFlhQdz4sE\n", + "0rTVpvYKkkMxhcdpmKFQHgKOw1TxhMiYNnPf7Ix5gLXrSob3fJXDhn/EhH2r6DO4mt696yjrV0/X\n", + "IXWU9amiT/cq+nStpG/pQPonNtOPJjIZ5iiJP4VOV7Y39mPzxtEseqeJks97ULuynq5PzOaghesY\n", + "KhcG0Z4N3v9fOYW/3vkkp1oNxmXtVdOagKXAH4APaL1UtvEcifSROEbDoEZKjl/LkF3ms8eYpYys\n", + "WMPQEcvZecRn7Fq+hqGJDQxkAwOpSTvpZdvKqdZ9qGooo25rD2q3llFXXUZdZTe2b+pK/YYuNKz/\n", + "iAnvLWb0Z8BHwGZISIEhOkSbG43+CbCBAX8axIYzgRpIZH/yuiXya2cmBUkm2ptpMRekILFAm1rs\n", + "kAWMHfUkpxz4GbseuYKd9m2kdOBydi77nDHUZTAWYSkNug9V2wexvnIIa1f3pGZrGXWbG+iyrI6y\n", + "JZrE6m10X7meQStWsNPqGnpthkR97j+hEIaGXYFPAbbSk3IzUasUJGlEkdjRwAsRpJMtKUhSFBG0\n", + "hWvoBuwBHAyMraRPxVJGjp3JEYPXMLTnEkb1WExF2WIqWMkIdCvfH7pQ39CHqspebF3TnW3L6+k6\n", + "bwMDZ22h9yJMc8EGzN0vuagtKKRfwKeQvPApsswL77bwp4CvSEHStmzu2graCdN38h1gFGTUeC0c\n", + "oyHxGocMW8Toc7pSf8L99Jv0Mbv3nMdE5rInq2l9oNoSGhnJ4q2jWLJ8ABtnr2Pw6x8x4cNNDJjX\n", + "QLfqjcidGCLeEuZR9hMxcy71TK0WYdlkShdMpn4POAbTZzIDM6/759GH1i6pkbCjE7vLqxzacyFj\n", + "d13H4JE19Nythp7D6yjrs43uveso672dbr220b18O916bqN7jyr69F/NsB5LqGg17RIa67tSv7mc\n", + "LXN7UDs/gV5ZTe83N9N/IbBcmppEMdDwQQ099uplhtuqhUTP9vZxnJUaye6YwuNsL4A/YAqSs4B5\n", + "UQYjdhQM/YDB9XQZtIjRO7/DfuM/Y9fxtfQYvJ5Bg7fSq7yGnj03069PBaP6bGRAq7e6ZmoUiz/d\n", + "Qvn8jQyYrSmZC8xtonTpNro0bYvigwkRX3KXajvaK0hew8xD8hLwQ+DvmEmuLqVj91X/EJgKO74G\n", + "z8NMlvXPwDbTgHOB/sCb3j4F+TyYNk2Co4EDXuXQsyvpe8AT9Oz7MN8peZ+9Wc8g6umWQUpJEhxO\n", + "H6qaelOmsnsmAAAYyElEQVTd0JvqmnK2VHdnW213ttV2Y3tNN7Zv6Up9VRcaKrvQsHk73T7/mN1n\n", + "fsDea4CNSxhdKHc3KaRfwKeQvPApOp4XgYJEF30rSDrtFSQHA28DdwDPRXC8ZcCVwGeYb97nYAqn\n", + "/YH3gaswhdS3MXdL/AzTkb8b9uaGj9RU7u/Xl8r96ii79BsMPnoNQ7sspoJFjEm7fXfqGsvZUjeI\n", + "9Rv7sXltD2rX92Lr2i40bCqlceMqhr/8BheUa+b/s5L+TZV5/jxCFIHqhPe9uYSmElvDncfZvsC9\n", + "wCZgMebCvgtmROAJER1jA6YGksCMNnx14L0yzLeBqWn2c/4b9I+564sn8eTsw3h52W7M39qHzS2m\n", + "YPVf3amtKafq9RIangT9EOgLQJfZ/gxCFDsNJ9dQpkHrEhoKoaXX2rWzB6ZPJIkpRJowNYv+nUiz\n", + "FDgD2IqpcYzx0p0c2u4ZzDD2Yc4XJOfy26+GC4xu1OldWFJ/ALPXK166FfTRoCeAbv/hCyGEFX5B\n", + "Ukp9Idxg4sS1cxxwC6b2sJ3sm7z2wjRT1WNqG8d76w/GFCQ7h7Z/qJVjOJEZbTmfewd8nRkvTOX+\n", + "e67klql38eNJbzO5o7dct0XlIM24UrYDcIiyHYBDVGd2rqW733Lg/HUnA3mf2CqdBcBPgOswhcB3\n", + "s9z/Y8zorH0xY3jNgDSTPjTX2gefjmlyA9gMvEeqQ015/1tbvp8fgHlg03+/H+ZmBSfiK9Bl2nm/\n", + "mJYnORaPzeVJndn/FZqAJJqDdCbbO7asMP3RkLpeFpwXMLMvjiZ909az3vthhfDNQAgRA5voe7PX\n", + "tFUI89BEfu20MUZWWCkmjkXAaswzKr4yzAjEsyzEJYQQAJTQVAh9IwXjFkzBUIHpK7kZaCTV/HMl\n", + "ponqZMxY4DOA5ZB2qFipkaQo2wE4RNkOwCHKdgAOUZ3ZuZLePw/cNNM9ophscaKPpDOGAn8EhgGV\n", + "mGdHvkRq0MfbMHeI3YO5I2w2poayNc9xCiFEUPDiWwf6p5D4hbVoRGSkRiKEyItqek0L3cpfCTqT\n", + "YSdcVJB9JEII4TTdfMqDRUAf4BRL4ThHCpLCoGwH4BBlOwCHKNsBOER1Zuee1DSeyN85jFcWlND4\n", + "hyjSLCRSkAghRDtKadJ/52ReYcq4e/jhNLNWWtd9UpAUhqTtAByStB2AQ5K2A3BIspP7tyg1xrKw\n", + "9ZnfiowUJEII0b6ZeMPJ+yMB96BWBlX1SEFSGJTtAByibAfgEGU7AIeozuycgDcwjy3s7q/bQnlt\n", + "J2MqGFKQCCFEBhJQm4BPqum9xHYsrpGCpDAkbQfgkKTtAByStB2AQ5JRJaRJ+HNbyWyJHilIhBAi\n", + "O3K7VogUJIVB2Q7AIcp2AA5RtgNwiIowLQ2QkPnbd5CCRAghOkYKEo8UJIUhaTsAhyRtB+CQpO0A\n", + "HJKMMC1p2gqRgkQIIbLjFyRSI/FIQVIYlO0AHKJsB+AQZTsAh6ioEkqgpUYSIgWJEEJ0jFw/PZIR\n", + "hSFpOwCHJG0H4JCk7QAckow6wYR0lewgBYkQQmRH+khCpCApDMp2AA5RtgNwiLIdgENUVAkF+kik\n", + "IPFIQSKEEFnQqfJDChKPFCSFIWk7AIckbQfgkKTtABySjDAt/8n2CJOMNylIhBAiC3L7b0v5Lkiu\n", + "Bt4GKoG1wNPAxDTbTQNWADWYCWUm5Cm+uFK2A3CIsh2AQ5TtAByiIkxL+khC8l2QTAH+H3AQcCTQ\n", + "ALwI9A9scxVwKXAhsD+mwHkBKM9rpEIIkZ4UJI7phSlMjveWE8AqTM3FV4aZ4nJqaF+pXgoh8u52\n", + "Lv0QtB7LZy/ajqWDIr922u4j6ePFsMlbHg0MBZ4PbFMHvAIcnN/QhBCiJb+PRDrbU2wXJHcD72Lm\n", + "QwYzJzLAmtB2awPviZaU7QAcomwH4BBlOwCHqKgS0iSkaSuki8Vj34GpZRxKZlWtdNtMBxZ7P28G\n", + "3iN1m5/y/pfl4lqmnfeLaXmSY/HYXJ4UVXqmRpJkC/ODfbtRxxvlsgLO8ZYXU0DuxNyVNT60fgzQ\n", + "BEwOrX8WeDi0TuqVQoi8+18ueQ+0HsenM23H0kEF0UdyN3A65q6tT0PvLQJWA8cE1pVhai2z8hKd\n", + "EEK0YQAbtwOsYvgeoEttx1OM7sE8Q3IEps/Df/UKbHMlppnqZGBPYAawPLQNSI0kSNkOwCHKdgAO\n", + "UbYDcIiKKqHVDPlXHzZr0Br0Y6CPiirtPIn9tbMJaPT+D75+FtruemAlUEvrDyTGPjMipGwH4BBl\n", + "OwCHKNsBOERFlZCGp+/ix35B4r++EFX6eSDXzgDJDCFE3ml4qgn0ldzyU9DzvYLkWNtxZUGunQGS\n", + "GUKIvNPwlFcNOQn0415BEn5g2mUF0dkuoqdsB+AQZTsAhyjbAThERZhW8DmS17yf948w/diRgkQI\n", + "IbITLEiqvJ+7WYrFCVKQFIak7QAckrQdgEOStgNwSDLCtIIFSaP3c1HfBiwFiRBCZCdYkDR4P9sc\n", + "JcQ6KUgKg7IdgEOU7QAcomwH4BAVYVpSkIRIQSKEENmRpq0QKUgKQ9J2AA5J2g7AIUnbATgkGWFa\n", + "UiMJkYJECCGyk65GIgWJiD1lOwCHKNsBOETZDsAhKsK00tVIpGlLCCFExqRpK0QKksKQtB2AQ5K2\n", + "A3BI0nYADklGmJZ0todIQSKEENmRGkmIFCSFQdkOwCHKdgAOUbYDcIiKMC3pbA+RgkQIIbIjne0h\n", + "UpAUhqTtAByStB2AQ5K2A3BIMsK0pGkrRAoSIYTIjjRthUhBUhiU7QAcomwH4BBlOwCHqAjT8guS\n", + "cUjTFiAFiRBCZOsN7/+Lf85Pd/V+LuoaSZzJVLtCiLzTkNDwmAb9GWOrval2F9mOKwty7QyQzBBC\n", + "WKGhh4YZS9jFm769aantmLIQ+2vn4cDTwHKgCfh2mm2mASuAGmAmMKGVtGKfGRFStgNwiLIdgEOU\n", + "7QAcoqJOUENiCbusB61LqV8Tdfo5FPm1M999JL2AD4CLgFpafqCrgEuBC4H9gbXAC0B5HmMUQoh2\n", + "JUAn0IsBGukyBHTCckhFqRo4O7CcAFYBVwfWlQFVwNQ0+0uNRAhh1UqGPen1kWjQv7IdT4ZiXyNp\n", + "y2hgKPB8YF0d8ApwsJWIhBCiDUNY+0pg8XLQp1kLxiKXCpJh3v/htsa1gfdEesp2AA5RtgNwiLId\n", + "gENULhItpelfd3BJcNUToC8D7dK1Neficu9za1Wx6cBi7+fNwHukhkJQ3v+yXFzLtPN+MS1Pciwe\n", + "m8uTcpF+ApKaux79Anef9X3uXbOY84cCt8PDBwD3RX28Di4r4BxveTEFJtxHMgZzJ9fk0HbPAg+n\n", + "2V/6SIQQ1mk4SoM+jccb/f6S/myo1qbf10UFde1M19m+kpad7ZXAuWn2L6jMEELEk4YSDY88y5d1\n", + "oONdn8mjp9qOrRWxv3b2wlQxJwFbgZ96P+/ivX8lponqZGBPYAbmmZNeadKKfWZESNkOwCHKdgAO\n", + "UbYDcIjKZeIaEg2U/GgxI/V3+f2OwmQg6/4D2rXHF2J/7VSY5qsmzKiZ/s8PBba5HlMzqUUeSMyU\n", + "sh2AQ5TtAByibAfgEJWPg2jo9T57XXIif6sqpV57Dytem49jZ0GunQGSGUIIJ2no/Ue+uR603pml\n", + "62zHEyLXzgDJDCGEs2Yy5Wt+E1cvqk+0HU+AXDsDJDNSlO0AHKJsB+AQZTsAh6j8H1KXBzvfQU/W\n", + "cJGGayzf0RX5tTMuz5EIIUTc1IaW33mdgzmEWQDrgAfyH5IIkxqJEMJpJ/HkB6Faif4T39BNZuEY\n", + "S2HJtTNAMkMI4TwN5X/mjKXBwuT3fFdruMxeSMInmZGibAfgEGU7AIco2wE4RNk8uIbjZvD1rX5B\n", + "0ptK/QqH/tJeONEqqoHFhBDChgT883SeKAe6jOeTZdX04RHO3sN2XEJqJEKIGJrK/S+C1oeTnG0p\n", + "BKmRCCFEnO3N+2sBttNtXw0Pa9jLdkzFTGokKcp2AA5RtgNwiLIdgEOU7QB8j3PacaB1Xzbpekq1\n", + "hkad3/ikRiKEEHF2Ok/8C/QnlfTjVP5a8yF7liCzwFojNRIhREzp08F/nETrc3iobh57jMzXwfN0\n", + "nFiQzBBCxJgeC00v+4XJMFbW9mXTzvk4cB6OERuSGSnKdgAOUbYDcIiyHYBDlO0A0tNnBB9UnMLM\n", + "Gfk4aNQJSh+JEELY8yTw4OG8/D7AcnY+SEN3Dadq2MlybEVBaiRCiILwKeMuLqFBd2Wbvp1L189n\n", + "N91IYouGn2joHvHh5NoZIJkhhCgIGiZOIdlscMeBrNPH8Jy+iDs3XMwdl0d7OOGTzEhRtgNwiLId\n", + "gEOU7QAcomwH0J6ZTDn2Rq6dM4k5/4Wm1eFRg6/i5qMiOpRcOwMkM1KU7QAcomwH4BBlOwCHKNsB\n", + "ZEcnQI/syZbTdmZpFWh9H+c9FVXiEaVTECQzhBAFbyir/uENPV+noV8ESRbNXVs/ABZhZhh7BzjU\n", + "bjhCCGHHGobNB/iY3bsD11mepjc2Tge2A98DdgN+DVQDu4S2kxpJirIdgEOU7QAcomwH4BBlO4CO\n", + "08eB1gka9aOcqTXcpaFPZxKMLDSHvQn8NrTuUyA8CUxRZEaGLrYdgEMkL1IkL1Jinhf6KtC6hAb9\n", + "CN/S2+myQcM1HSxQIr92dok6wU7qBuwL3BZa/zwyqFlbomg3LRSSFymSFykxz4vEraDLmiiddjaP\n", + "ch6/HTCZ/960L3N+eh1Vf1vI2Fu/ytPVi6kYV0WfUU/z1VfnsdfHtqO2ZQTQRMs+kZ8B4UyRGknK\n", + "NNsBOGSa7QAcMs12AA6ZZjuAztMJ0JdD04LwrcHh104se6ithKKOzLUaieiYCtsBOKTCdgAOqbAd\n", + "gEMqbAfQeQkN3G5eelAJjfvvy5wzurPtK/PZo38NPenPpoa+VG7TJNbnNbJ8HiwD3YCtwBnAXwPr\n", + "7wEmAEcE1i0AxuYvNCGEKAgLgXG2g8i12aTvbL/JQixCCCFi6OvANsztv3sAdwNVtLz9VwghhGjV\n", + "BZgHEuuAt5EHEoUQQgghhBC5ls2wKBOAmcBqb/uFmD6UroFtFOZW4vBrfMRx50JHh4jZFTMKQHWa\n", + "96YA/yWVX+d1Psy8iDovFMVxXlSQ/nMeE9quGM6LCtrPC9XKNq6fFx35+7gY8zhFHbASuDn0flzP\n", + "iYyHRfGNBc4G9vK2+QqmUPlVYBuFORF2B4YEXq6OL+bLNi983TC//GcwfUpBozF3w93tpfl97xin\n", + "RBZ1buQiLxTFcV5UYD7n0TT/nMEvW8VyXlTQfl4o4ndedOTv4w7gE8w1swLYG/hS4P24nhNA5sOi\n", + "tOUOYFZgWWFOjIGdiiz/OpoXdwIPAt+m5bfwWzEnT9ADNM8vF+UiLxTFcV5UYD7n5DbSLJbzooL2\n", + "80IRv/Mi23zYDVMo7NZGmh06J1wobf1hUZ4Prc9mWJRxwLFp0gBT3VsJvIj7A7d1NC+O914/Iv2z\n", + "QQe1kuZ+QGmHIs29XOWFrxjOCzBzgq8BXgNODb1XTOcFtJ0XvricFx3JhxOBz4HjvP8XAdOBwYFt\n", + "OnROuFCQDMIEuCa0fi0wrJ19Z2Ha8T7FlM7TAu+tBM7HVMlOwZSy/8HtO8A6khcjgN8BZwI1rWwz\n", + "NE2aazAjGwzqUKS5l6u8KJbzohq4DDgN+DLmMz6OyRtfsZwXmeRF3M6LjuTDGGAU5hGLs4GzME15\n", + "/whs06FzIu5DpHwdKAcmYfpHbgOu9N771Hv5ZmOquFdgvpEUikeB+zC3SRe7TPKiWM6LDZgmPt8c\n", + "TLPNlcCfrERkTyZ5UQznRQnQHVOALPDWnYUpNPenE9cQF2ok64FGTEkYNBRY1c6+yzF3H8wAfgJc\n", + "RNtV8rcwd/O4qiN5cQRwPVDvvX4P9PJ+/r63zWpafksZCjR4x3RRrvIinUI8L9J5m+afs1jOi3TC\n", + "eZGOy+dFR/JhFeZ3uyCwboGXzkhvuUPnhAsFyXbMHTbh2xKPJrtOv1LM52nrM03CVGFd1ZG82BNz\n", + "54X/+hmmuW9v4P+8bd7w0gin+TbmJHJRrvIinUI8L9IJf85iOS/SyeR37vJ50ZF8eA3TCjUmsG4M\n", + "5tq5xFuO4zmxQ3vDotyM6fzynQV8DdO+N8bbfznwx8A2F2M6l3YFJnppNAEn5epDRCTbvAg7h5Z3\n", + "KlUAWzDV+z0w3863ASdHFHOu5CIviuW8+DbwDW/b3YDLvf0vCmxTQXGcF5nkRRzPi2zzIYG5mSCJ\n", + "KST3AV6mecFTQTzPiR3aGhblYcxdBr4zMKVxFeZCMRfTtNU9sM0VmDbPGkwb6cs0v1/aZdnkRdg5\n", + "tHx2AuBwTJ7VYR4ymhpFoHkQdV4Uy3lxNjAPc1GoxDTTfDNNmsVwXmSSF3E9L7L9+xgGPIH5u1iD\n", + "6VccHNomrueEEEIIIYQQQgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEKIIPYN5mMqXxEza0xnTaT7K\n", + "qUuasDNx0HQ6nycVmPj3bWOb/bxtRraxjYixuI/+KwqT9l6+kzADL2ZCAS9hhrzeGFjf3vwk+TAN\n", + "Mw/GXpbj8IXzWYgOkYJE5Eo3zMByUdjcgX3ChUa6eezjrAtmRNbOsF2wigLhwui/wn1JzDwfd2O+\n", + "5W/EzP0SvBAtxgzh/hCwCTOGD5jZ2l7GzAO9HLgX6B3YryemiaUaM4T11d76YNpJ4DeB5W6Y6UQX\n", + "kxoP6EeYSXte8rZZh2lOechbnk7zZpzuwF3eMWsxo54eEnhfefsfiZk0bStmLKN9Atv09T7nGi+N\n", + "hTQfCDDoHMxoxBO9dJswY0D5BgJ/wYwHtZDmky5VeNuf4X2+GlLjH30H+Mg7/ieYwQeDeXceZgyp\n", + "WkyePEfq7157216E+d1sxORXj8D+7eVTOl/CTO9QC7wCjA+9n02+CSEKRBIzyNvdmIvCaZhawiWB\n", + "bRZjBsS7HDMi81hME061t91Y4ADMSKN/Cex3L+YidjTmIvuEl85DgW1m0ryP5DFgGWZE0grMQHVn\n", + "Yi6QJ2MuursDQ0gVWg8DTwfSuBszRPiXMSPC/s6L1Z+LQXnpzAameNs8h7lo+34DvIvpA9jF2+5r\n", + "pFeGmXxtvhfXEFKDjDZ5n+ebmLz7JWbEVX8U1wpvm0WYvpRRwE7Aud5n8NedgJlz4ofefvthmgS/\n", + "4aX1BZrP2TMd83v8rff5jsZ8CfhJFvnkx+b3keyCKdyD58pyms95kU2+CSEKRBLzDTPoWszFz7cY\n", + "eCq0zSOYyaWCJmEuPIMws1vWYS50vl6Yi1lrBcmu3v7heRh8ynt/QGj9dFI1kl6YC/W3Au+XYCb5\n", + "uTGUTnBuhoO9dSO85aeAB1uJI51pwIdp1jcBNwWWSzE1IH+E2gpvm0ua78ZSmtdcwNRI5nk/n4Ip\n", + "KMpbiWc6Zh6KYA3md8AL3s+Z5JMfm1+Q/JL050qwsz3bfBOOk6YtkQmN+WYeNBvzrbg8sM07oW0m\n", + "Yy5C1YHXa962Y71XN0xziW8r6S+2vn0wF6WZ2X6IgLFAV+D1wLomL44JoW0/CPzszzw3xPv/PuB0\n", + "4D1MbePwTsQUPE4jphlqSGibYP4OBnYmVUPwXzeTmrjoeUxBsQgzV8/ZtCxUPqJ5h/uqwHGzySff\n", + "HqQ/V4KizDfhAClIRKYy6ZjdmmafB2g+a+EXMLWK9zt5rFxIYC6UQcG7xfwLrv938xymSel2TA3r\n", + "WZrXpLIRvitN0/LvM5i//nvn0Tx/J3ovMP0t+2ImQFqK6X/6GBgeSCfcYZ/uuGHp8im4f3u/vyjz\n", + "TThAChKRiQRwYGjdF4EVmItVa+Zgpr/9PM3L7ySvBw4K7NPL26c172HO2yNbed+/U6y0lffxjrud\n", + "5pMAlXpxfJR2j9ZtwHzb/w5mNrlvY77FtxZbW3FlYw2m72Ic6fPX14ipvV2DKcR7AccH3m/r9t+O\n", + "5NN80p8rYdnkm3Cc3P4rMjUCc/fOfZhO9MtJtZO35lZMs8Z9pJpgdsd0Cp+PKYQe9LZbh2lW+Rkt\n", + "v+AkSH3L/RTTIf97TMfxu5gmnlGYC9MSzMXxBMyDjTW0rClt9WK6FViP6d+5BNNcdG87nyno55iZ\n", + "5D7C/C2dQqpwTGeRF+c+mP6lKjp3i/T1mI7rzcC/MBfifTG/q1sweTAWc+fURuAIzM0H8wNptFV7\n", + "6Eg+3Q9cRvNz5bzQNtnmmxCiAMzEXDh+g+kI34hp2w5e8BcBl6bZdzLmIleJKTg+wHQ6+3oCfyB1\n", + "+++1mE7xtu7a6oa5uC3H1GwWAD8IvH8d5tt6YyCd8F1b3TDzUq/20piF6Uz3KW//YKd9hbfO71i+\n", + "BjPN81bMN+xnMHc2taYb5o61jTS//Tfdk+3B/AwfN8ifdrrWS/cVTFMWmNt0X8IUAjWYvP92YN9w\n", + "noApnIL9Ne3lU7rYjiN1+++rmJsGgndtZZtvQogCEL6QCyHEDtJHIjIRbFoSQohmpCARmZAxmYQQ\n", + "QgghhBBCCCGEEEIIIYQQQgghhBBCCCGEEEKI+Pv/8Ee54ouRNgMAAAAASUVORK5CYII=\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "report.metrics_vs_cut(AMS, metric_label='AMS').plot()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 2", + "language": "python", + "name": "python2" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.8" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}