pandasvault.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "r93Uya5Mgx3Q"
   },
   "source": [
    "# **PandasVault** ⁠— Advanced Pandas Functions and Code Snippets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uZgKLm4RW9yV"
   },
   "source": [
    "[`Github PandasVault Link`](https://github.com/firmai/pandasvault), [`LinkedIn`](https://www.linkedin.com/company/firmai)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 350
    },
    "colab_type": "code",
    "id": "UgxlmvkaVPJA",
    "outputId": "dd35e8cc-87da-4b51-c972-c08fbaf30ae3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting pandasvault\n",
      "  Downloading https://files.pythonhosted.org/packages/66/70/70f2679fd15f13cc418bd8b5bc69c68a344315d75a6a501a6606fa5ba1b7/pandasvault-0.0.3.tar.gz\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from pandasvault) (0.25.3)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pandasvault) (1.17.5)\n",
      "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (from pandasvault) (0.0)\n",
      "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas->pandasvault) (2.6.1)\n",
      "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->pandasvault) (2018.9)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sklearn->pandasvault) (0.22.1)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.6.1->pandas->pandasvault) (1.12.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn->pandasvault) (0.14.1)\n",
      "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn->pandasvault) (1.4.1)\n",
      "Building wheels for collected packages: pandasvault\n",
      "  Building wheel for pandasvault (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for pandasvault: filename=pandasvault-0.0.3-cp36-none-any.whl size=6493 sha256=3f67660b7407963b897e4e0694704c831f64d4c205bd112289e563c417f248f4\n",
      "  Stored in directory: /root/.cache/pip/wheels/ae/6a/ee/1f31529eeba2dda5cffc612d17d4395ceadd1d8c51bec99688\n",
      "Successfully built pandasvault\n",
      "Installing collected packages: pandasvault\n",
      "Successfully installed pandasvault-0.0.3\n"
     ]
    }
   ],
   "source": [
    "!pip install pandasvault"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3C6Mh8LpTY1J"
   },
   "source": [
    "### List of Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zSgTh5gATehT"
   },
   "source": [
    "```python\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pandasvault as pv\n",
    "\n",
    "\"\"\"TABLE PROCESSING\"\"\"\n",
    "df = pv.list_shuff([\"target\",\"c\",\"d\"],df)\n",
    "df = pv.reduce_mem_usage(df)\n",
    "\n",
    "\"\"\"TABLE EXPLORATION\"\"\"\n",
    "df = pv.corr_list(df)\n",
    "df = pv.missing_data(df)\n",
    "\n",
    "\"\"\"FEATURE PROCESSING\"\"\"\n",
    "df = pv.drop_corr(df, thresh=0.1,keep_cols=[\"target\"])\n",
    "df = pv.replace_small_cat(df,[\"cat\"])\n",
    "qconstant_col = pv.constant_feature_detect(data=df,threshold=0.9)\n",
    "df_train, scl = pv.scaler(df,target=\"target\",cols_ignore=[\"a\"],type=\"MinMax\")\n",
    "df_test = pv.scaler(df_test,scaler=scl,train=False, target=\"target\",cols_ignore=[\"a\"])\n",
    "df = pv.impute_null_with_tail(df,cols=df.columns)\n",
    "index,para = pv.outlier_detect(df,\"a\",threshold=0.5,method=\"IQR\")\n",
    "df = pv.windsorization(data=df,col='a',para=para,strategy='both')\n",
    "df = pv.impute_outlier(data=df,col='a', outlier_index=index,strategy='mean')\n",
    "\n",
    "\"\"\"FEATURE EXTRACTION\"\"\"\n",
    "df = pv.auto_dummy(df, unique=3)\n",
    "df = pv.binarise_empty(df, frac=0.6)\n",
    "df = pv.polynomials(df, [\"a\",\"b\"]) \n",
    "df = pv.transformations(df,[\"a\",\"b\"])\n",
    "df = pv.pca_feature(df,variance_or_components=0.80,drop_cols=[\"target\",\"a\"])\n",
    "df = pv.multiple_lags(df, start=1, end=2,columns=[\"a\",\"target\"])\n",
    "df = pv.multiple_rolling(df, columns=[\"a\"])\n",
    "df = pv.date_features(df, date=\"date_fake\")\n",
    "df['distance_central'] = df.apply(pv.haversine_distance,axis=1)\n",
    "\n",
    "\"\"\"MODEL VALIDATION\"\"\"\n",
    "scores = pv.classification_scores(y_test, y_predict, y_prob)\n",
    "\n",
    "```\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MO334lbpZ7Uw"
   },
   "source": [
    "## List of Code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Wv5P8ZHUcHSi"
   },
   "source": [
    "\n",
    "#### [Table Processing](#table-processing)\n",
    "  -\t[<font color=\"black\">Configure Pandas</font>](#configure-pandas)\n",
    "  -\t[<font color=\"black\">Data Frame Formatting</font>](#data-frame-formatting)\n",
    "  -\t[<font color=\"black\">Data Frames for Testing</font>](#data-frames-for-testing)\n",
    "  -\t[<font color=\"black\">Lower Case Columns</font>](#lower-case-columns)\n",
    "  -\t[<font color=\"black\">Front and Back Column Selection</font>](#front-and-back-columns)\n",
    "  -\t[<font color=\"black\">Fast Data Frame Split</font>](#fast-data-frame-split)\n",
    "  -\t[<font color=\"black\">Create Features and Labels List</font>](#create-features-and-labels-list)\n",
    "  -\t[<font color=\"black\">Short Basic Commands</font>](#short-basic-commands)\n",
    "  -\t[<font color=\"black\">Read Commands</font>](#read-commands)\n",
    "  -\t[<font color=\"black\">Create Ordered Categories</font>](#create-ordered-categories)\n",
    "  -\t[<font color=\"black\">Select Columns Based on Regex</font>](#select-columns-based-on-regex)\n",
    "  -\t[<font color=\"black\">Accessing Group of Groupby Object</font>](#accessing-group-of-groupby-object)\n",
    "  -\t[<font color=\"black\">Multiple External Selection Criteria</font>](#multiple-external-selection-criteria)\n",
    "  -\t[<font color=\"black\">Memory Reduction Script</font>](#memory-reduction-script)\n",
    "  -\t[<font color=\"black\">Verify Primary Key</font>](#verify-primary-key)\n",
    "  -\t[<font color=\"black\">Shift Columns to Front</font>](#shift-columns-to-front)\n",
    "  -\t[<font color=\"black\">Multiple Column Assignment</font>](#multiple-column-assignment)\n",
    "  -\t[<font color=\"black\">Method Changing Technique</font>](#method-chaning-event)\n",
    "  -\t[<font color=\"black\">Load Multiple Files</font>](#load-multiple-files)\n",
    "  -\t[<font color=\"black\">Drop Rows and Column Substring</font>](#drop-rows-and-column-substring)\n",
    "  -\t[<font color=\"black\">Explode a Column</font>](#explode-a-column)\n",
    "  -\t[<font color=\"black\">Nest List Back into Column</font>](#nest-list-back-into-column)\n",
    "  -\t[<font color=\"black\">Split Cells with List</font>](#split-cells-with-list)\n",
    "\n",
    "\n",
    "#### [Table Exploration](#table-exploration)\n",
    "\n",
    "  -\t[<font color=\"black\">Groupby Functionality</font>](#groupby-functionality)\n",
    "  -\t[<font color=\"black\">Cross Correlation Series Without Duplicates](#cross-correlation-series-without-duplicates)\n",
    "  -\t[<font color=\"black\">Missing Data Report</font>](#missing-data-report)\n",
    "  -\t[<font color=\"black\">Duplicated Rows Report</font>](#duplicated-rows-report)\n",
    "  -\t[<font color=\"black\">Skewness</font>](#skewness)\n",
    "\n",
    "\n",
    "#### [Feature Processing](#feature-processing)\n",
    "\n",
    "  -\t[<font color=\"black\">Remove Correlated Pairs</font>](#remove-correlated-pairs)\n",
    "  -\t[<font color=\"black\">Replace Infrequently Occurring Categories</font>](#replace-infrequently-occuring-categories)\n",
    "  -\t[<font color=\"black\">Quasi-Constant Feature Detection</font>](#quasi-constant-feature-detection)\n",
    "  -\t[<font color=\"black\">Filling Missing Values Separately</font>](#filling-missing-values-separately)\n",
    "  -\t[<font color=\"black\">Conditioned Column Value Replacement</font>](#conditioned-column-value-replacement)\n",
    "  -\t[<font color=\"black\">Remove Non-numeric Values in Data Frame</font>](#remove-non-numeric-values-in-data-frame)\n",
    "  -\t[<font color=\"black\">Feature Scaling, Normalisation, Standardisation</font>](#feature-scaling-normalisation-standardisation)\n",
    "  -\t[<font color=\"black\">Impute Null with Tail Distribution</font>](#impute-null-with-tail-distribution)\n",
    "  -\t[<font color=\"black\">Detect Outliers</font>](#detect-outliers)\n",
    "  -\t[<font color=\"black\">Windzorise Outliers</font>](#windsorize-outliers)\n",
    "  -\t[<font color=\"black\">Drop Outliers</font>](#drop-outliers)\n",
    "  -\t[<font color=\"black\">Impute Outliers</font>](#impute-outliers)\n",
    "\n",
    "#### [Feature Engineering](#feature-engineering)\n",
    "\n",
    "  -\t[<font color=\"black\">Automated Dummy Encoding</font>](#automate-dummy-encodings)\n",
    "  -\t[<font color=\"black\">Binarise Empty Columns</font>](#binarise-empty-columns)\n",
    "  -\t[<font color=\"black\">Polynomials</font>](#polynomials)\n",
    "  -\t[<font color=\"black\">Transformations</font>](#transformations)\n",
    "  -\t[<font color=\"black\">Genetic Programming</font>](#genetic-programming)\n",
    "  -\t[<font color=\"black\">Principal Component</font>](#principal-component)\n",
    "  -\t[<font color=\"black\">Multiple Lags</font>](#multiple-lags)\n",
    "  -\t[<font color=\"black\">Multiple Rolling</font>](#multiple-rolling)\n",
    "  -\t[<font color=\"black\">Date Features</font>](#data-features)\n",
    "  -\t[<font color=\"black\">Haversine Distance</font>](#havervsine-distance)\n",
    "  -\t[<font color=\"black\">Parse Address</font>](#parse-address)\n",
    "  -\t[<font color=\"black\">Processing Strings in Pandas</font>](#processing-strings-in-pandas)\n",
    "  -\t[<font color=\"black\">Filtering Strings in Pandas</font>](#filtering-strings-in-pandas)\n",
    "\n",
    "\n",
    "#### [Model Validation](#model-validation)\n",
    "\n",
    "  -\t[<font color=\"black\">Classification Metrics</font>](#classification-metrics)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8dXsYrjUlpU3"
   },
   "source": [
    "*If you are running the code <mark>for the first time load this</mark> test dataframe:*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 211
    },
    "colab_type": "code",
    "id": "o2EWybtEVZpL",
    "outputId": "0823574c-68ef-480b-896c-3bc9d48aeba0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pandasvault in /usr/local/lib/python3.6/dist-packages (0.0.3)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pandasvault) (1.17.5)\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from pandasvault) (0.25.3)\n",
      "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (from pandasvault) (0.0)\n",
      "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas->pandasvault) (2018.9)\n",
      "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas->pandasvault) (2.6.1)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sklearn->pandasvault) (0.22.1)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.6.1->pandas->pandasvault) (1.12.0)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn->pandasvault) (0.14.1)\n",
      "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn->pandasvault) (1.4.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install pandasvault"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "blGCV6F_lvto"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pandasvault as pv\n",
    "\n",
    "np.random.seed(1)\n",
    "\"\"\"quick way to create a data frame for testing\"\"\" \n",
    "df_test = pd.DataFrame(np.random.randn(3, 4), columns=['a', 'b', 'c', 'd']) \\\n",
    "    .assign(target=lambda x: (x['b']+x['a']/x['d'])*x['c'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VO6NxvYjofas"
   },
   "source": [
    "<a name=\"table-processing\"></a>\n",
    "\n",
    "### **Table Processing**\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XXzvo89dzUPd"
   },
   "source": [
    "<a name=\"configure-pandas\"></a>\n",
    "**>>> Configure Pandas (func)**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ATVzkhhqzWGE"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "def pd_config():\n",
    "    options = {\n",
    "        'display': {\n",
    "            'max_colwidth': 25,\n",
    "            'expand_frame_repr': False,  # Don't wrap to multiple pages\n",
    "            'max_rows': 14,\n",
    "            'max_seq_items': 50,         # Max length of printed sequence\n",
    "            'precision': 4,\n",
    "            'show_dimensions': False\n",
    "        },\n",
    "        'mode': {\n",
    "            'chained_assignment': None   # Controls SettingWithCopyWarning\n",
    "        }\n",
    "    }\n",
    "\n",
    "    for category, option in options.items():\n",
    "        for op, value in option.items():\n",
    "            pd.set_option(f'{category}.{op}', value)  # Python 3.6+\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    pv.pd_config()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "efMmqu9q3XDg"
   },
   "source": [
    "<a name=\"data-frame-formatting\"></a>\n",
    "\n",
    "**>>> Data Frame Formatting**\n",
    "\n",
    "---\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DTmrcoOB4JvS"
   },
   "outputs": [],
   "source": [
    "df = df_test.copy()\n",
    "df[\"number\"] = [3,10,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "HY61atQp3dDO",
    "outputId": "7efe485f-f274-447a-a0a5-9af574fda052"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_3b06850c_627f_11ea_a8db_0242ac1c0002row0_col0 {\n",
       "            : ;\n",
       "            background-color:  green;\n",
       "        }    #T_3b06850c_627f_11ea_a8db_0242ac1c0002row0_col4 {\n",
       "            background-color:  #08306b;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_3b06850c_627f_11ea_a8db_0242ac1c0002row0_col5 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg, transparent 50.0%, lightblue 50.0%, lightblue 65.0%, transparent 65.0%);\n",
       "        }    #T_3b06850c_627f_11ea_a8db_0242ac1c0002row1_col4 {\n",
       "            background-color:  #f7fbff;\n",
       "            color:  #000000;\n",
       "        }    #T_3b06850c_627f_11ea_a8db_0242ac1c0002row1_col5 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg, transparent 50.0%, lightblue 50.0%, lightblue 100.0%, transparent 100.0%);\n",
       "        }    #T_3b06850c_627f_11ea_a8db_0242ac1c0002row2_col0 {\n",
       "            background-color:  red;\n",
       "            : ;\n",
       "        }    #T_3b06850c_627f_11ea_a8db_0242ac1c0002row2_col4 {\n",
       "            background-color:  #1f6eb3;\n",
       "            color:  #f1f1f1;\n",
       "        }    #T_3b06850c_627f_11ea_a8db_0242ac1c0002row2_col5 {\n",
       "            width:  10em;\n",
       "             height:  80%;\n",
       "            background:  linear-gradient(90deg, transparent 50.0%, lightblue 50.0%, lightblue 55.0%, transparent 55.0%);\n",
       "        }</style><table id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002\" ><caption>DF with different stylings</caption><thead>    <tr>        <th class=\"col_heading level0 col0\" >a</th>        <th class=\"col_heading level0 col1\" >b</th>        <th class=\"col_heading level0 col2\" >c</th>        <th class=\"col_heading level0 col3\" >d</th>        <th class=\"col_heading level0 col4\" >target</th>        <th class=\"col_heading level0 col5\" >number</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                                <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row0_col0\" class=\"data row0 col0\" >$1.62</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row0_col1\" class=\"data row0 col1\" >-0.6118</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row0_col2\" class=\"data row0 col2\" >-0.5282</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row0_col3\" class=\"data row0 col3\" >-1.073</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row0_col4\" class=\"data row0 col4\" >$1.12270</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row0_col5\" class=\"data row0 col5\" >3</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row1_col0\" class=\"data row1 col0\" >$0.87</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row1_col1\" class=\"data row1 col1\" >-2.302</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row1_col2\" class=\"data row1 col2\" >1.745</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row1_col3\" class=\"data row1 col3\" >-0.7612</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row1_col4\" class=\"data row1 col4\" >$-5.99941</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row1_col5\" class=\"data row1 col5\" >10</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                                <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row2_col0\" class=\"data row2 col0\" >$0.32</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row2_col1\" class=\"data row2 col1\" >-0.2494</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row2_col2\" class=\"data row2 col2\" >1.462</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row2_col3\" class=\"data row2 col3\" >-2.06</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row2_col4\" class=\"data row2 col4\" >$-0.59103</td>\n",
       "                        <td id=\"T_3b06850c_627f_11ea_a8db_0242ac1c0002row2_col5\" class=\"data row2 col5\" >1</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fba61594240>"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df_out = (\n",
    "  df.style.format({\"a\":\"${:.2f}\", \"target\":\"${:.5f}\"})\n",
    " .hide_index()\n",
    " .highlight_min(\"a\", color =\"red\")\n",
    " .highlight_max(\"a\", color =\"green\")\n",
    " .background_gradient(subset = \"target\", cmap =\"Blues\")\n",
    " .bar(\"number\", color = \"lightblue\", align = \"zero\")\n",
    " .set_caption(\"DF with different stylings\")\n",
    ") ; df_out\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "56hWCGiq-Tje"
   },
   "source": [
    "<a name=\"data-frame-formatting\"></a>\n",
    "\n",
    "**>>> Data Frames For Testing**\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 258
    },
    "colab_type": "code",
    "id": "JTnxutE3-Z4P",
    "outputId": "7a36d2ac-be53-4b78-f0ae-3a255cb67a9f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Contains missing values\n",
      "Contains datetime values\n",
      "Contains mixed values\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>foo1</td>\n",
       "      <td>2009-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>foo2</td>\n",
       "      <td>2009-01-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>foo3</td>\n",
       "      <td>2009-01-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>foo4</td>\n",
       "      <td>2009-01-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>foo5</td>\n",
       "      <td>2009-01-07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     A    B     C          D\n",
       "0  0.0  0.0  foo1 2009-01-01\n",
       "1  1.0  1.0  foo2 2009-01-02\n",
       "2  2.0  0.0  foo3 2009-01-05\n",
       "3  3.0  1.0  foo4 2009-01-06\n",
       "4  4.0  0.0  foo5 2009-01-07"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = pd.util.testing.makeDataFrame() # contains random values\n",
    "print(\"Contains missing values\")\n",
    "df2 = pd.util.testing.makeMissingDataframe() # contains missing values\n",
    "print(\"Contains datetime values\")\n",
    "df3 = pd.util.testing.makeTimeDataFrame() # contains datetime values\n",
    "print(\"Contains mixed values\")\n",
    "df4 = pd.util.testing.makeMixedDataFrame(); df4.head() # contains mixed values\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aUCr496n_p4d"
   },
   "source": [
    "<a name=\"lower-case-columns\"></a>\n",
    "\n",
    "\n",
    "**>>> Lower Case Columns**\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WcnfBPjE_qn-"
   },
   "outputs": [],
   "source": [
    "## Lower-case all DataFrame column names \n",
    "df = df_test.copy() ; df\n",
    "df.columns = [\"A\",\"BGs\",\"c\",\"dag\",\"Target\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "1HzN6r9tjbQs",
    "outputId": "19dadae8-a345-4910-fd44-b4a2af2c79d3"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>bgs</th>\n",
       "      <th>c</th>\n",
       "      <th>dag</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a     bgs       c     dag  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 9,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns = map(str.lower, df.columns); df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "bGvl6lH2TApx"
   },
   "source": [
    "<a name=\"front-and-back-columns\"></a>\n",
    "\n",
    "**>>> Front and Back Column Selection**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "clBkz0w2TIaZ",
    "outputId": "06b0f390-12f2-4149-889f-bc00e0c33421"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "gaC7sPxWTIOO",
    "outputId": "deda9ad9-cd28-4aec-a0c0-2d25073d7dd5"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        d  target\n",
       "0 -1.0730  1.1227\n",
       "1 -0.7612 -5.9994\n",
       "2 -2.0601 -0.5910"
      ]
     },
     "execution_count": 11,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def front(self, n):\n",
    "    return self.iloc[:, :n]\n",
    "\n",
    "def back(self, n):\n",
    "    return self.iloc[:, -n:]\n",
    "\n",
    "pd.back = back\n",
    "pd.front = front\n",
    "\n",
    "pd.back(df,2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "49ntNtsA13uq"
   },
   "source": [
    "<a name=\"fast-data-frame-split\"></a>\n",
    "\n",
    "**>>> Fast Data Frame Split**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "sMnFTkat2BJ7",
    "outputId": "df95310c-0ba8-4256-c9b0-daedbe3e8ad5"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 12,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 112
    },
    "colab_type": "code",
    "id": "CVXX3GOc15aH",
    "outputId": "e7149f8f-1521-44a7-8de7-5508f17e3332"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994"
      ]
     },
     "execution_count": 13,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test =  df.sample(frac=0.4)\n",
    "train = df[~df.isin(test)].dropna(); train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AF-MUEy1_FaH"
   },
   "source": [
    "<a name=\"create-features-and-labels-list\"></a>\n",
    "\n",
    "**>>> Create Features and Labels List**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 52
    },
    "colab_type": "code",
    "id": "wU3u0kHX_K_u",
    "outputId": "87abb0fe-8754-4726-8868-46061de2b89d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y = target\n",
      "X = ['a', 'b', 'c']\n"
     ]
    }
   ],
   "source": [
    "df = df_test.head()\n",
    "y = 'target'\n",
    "X = [name for name in df.columns if name not in [y, 'd']]\n",
    "print('y =', y)\n",
    "print('X =', X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "bkdeSO4x1p6b"
   },
   "source": [
    "<a name=\"short-basic-commands\"></a>\n",
    "\n",
    "**>>> Short Basic Commands**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "id": "Ub8jxAy52E72",
    "outputId": "c68c94b0-934a-426c-dc57-3197b08d5b82"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>category</th>\n",
       "      <th>k</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>1</td>\n",
       "      <td>1: -1.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>0</td>\n",
       "      <td>0: -0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>0</td>\n",
       "      <td>0: -2.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>1</td>\n",
       "      <td>1: -1.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>0</td>\n",
       "      <td>0: -0.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target category        k\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227        1  1: -1.1\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994        0  0: -0.8\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910        0  0: -2.1\n",
       "3  1.6243 -0.6118 -0.5282 -1.0730  1.1227        1  1: -1.1\n",
       "4  0.8654 -2.3015  1.7448 -0.7612 -5.9994        0  0: -0.8"
      ]
     },
     "execution_count": 15,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df[\"category\"] = np.where( df[\"target\"]>1, \"1\",  \"0\")\n",
    "df[\"k\"] = df[\"category\"].astype(str) +\": \" + df[\"d\"].round(1).astype(str) \n",
    "df = df.append(df, ignore_index=True) ; df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "id": "sAXvryBr0f6a",
    "outputId": "d3a303c1-fb9d-4911-a8d1-5fe137d7e2e2"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>category</th>\n",
       "      <th>k</th>\n",
       "      <th>new1</th>\n",
       "      <th>new2</th>\n",
       "      <th>new3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19.0</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>1</td>\n",
       "      <td>1: -1.1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.1</td>\n",
       "      <td>18.4718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19.0</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>0</td>\n",
       "      <td>0: -0.8</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.8</td>\n",
       "      <td>20.7448</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>19.0</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>0</td>\n",
       "      <td>0: -2.1</td>\n",
       "      <td>0</td>\n",
       "      <td>-2.1</td>\n",
       "      <td>20.4621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>19.0</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>1</td>\n",
       "      <td>1: -1.1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.1</td>\n",
       "      <td>18.4718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19.0</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>0</td>\n",
       "      <td>0: -0.8</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.8</td>\n",
       "      <td>20.7448</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      a       b       c       d  target category        k new1  new2     new3\n",
       "0  19.0 -0.6118 -0.5282 -1.0730  1.1227        1  1: -1.1    1  -1.1  18.4718\n",
       "1  19.0 -2.3015  1.7448 -0.7612 -5.9994        0  0: -0.8    0  -0.8  20.7448\n",
       "2  19.0 -0.2494  1.4621 -2.0601 -0.5910        0  0: -2.1    0  -2.1  20.4621\n",
       "3  19.0 -0.6118 -0.5282 -1.0730  1.1227        1  1: -1.1    1  -1.1  18.4718\n",
       "4  19.0 -2.3015  1.7448 -0.7612 -5.9994        0  0: -0.8    0  -0.8  20.7448"
      ]
     },
     "execution_count": 16,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"set display width, col_width etc for interactive pandas session\"\"\" \n",
    "pd.set_option('display.width', 200)\n",
    "pd.set_option('display.max_colwidth', 20)\n",
    "pd.set_option('display.max_rows', 100)\n",
    "           \n",
    "\"\"\"when you have an excel sheet with spaces in column names\"\"\"\n",
    "df.columns = [c.lower().replace(' ', '_') for c in df.columns]\n",
    "\n",
    "\"\"\"Add prefix to all columns\"\"\"\n",
    "df.add_prefix(\"1_\")\n",
    "\n",
    "\"\"\"Add suffix to all columns\"\"\"\n",
    "df.add_suffix(\"_Z\")\n",
    "\n",
    "\"\"\"Droping column where missing values are above a threshold\"\"\"\n",
    "df.dropna(thresh = len(df)*0.95, axis = \"columns\") \n",
    "\n",
    "\"\"\"Given a dataframe df to filter by a series [\"a\",\"b\"]:\"\"\" \n",
    "df[df['category'].isin([\"1\",\"0\"])]\n",
    "\n",
    "\"\"\"filter by multiple conditions in a dataframe df\"\"\"\n",
    "df[(df['a'] >1) & (df['b'] <1)]\n",
    "\n",
    "\"\"\"filter by conditions and the condition on row labels(index)\"\"\"\n",
    "df[(df.a > 0) & (df.index.isin([0, 1]))]\n",
    "\n",
    "\"\"\"regexp filters on strings (vectorized), use .* instead of *\"\"\"\n",
    "df[df.category.str.contains(r'.*[0-9].*')]\n",
    "\n",
    "\"\"\"logical NOT is like this\"\"\"\n",
    "df[~df.category.str.contains(r'.*[0-9].*')]\n",
    "\n",
    "\"\"\"creating complex filters using functions on rows\"\"\"\n",
    "df[df.apply(lambda x: x['b'] > x['c'], axis=1)]\n",
    "\n",
    "\"\"\"Pandas replace operation\"\"\"\n",
    "df[\"a\"].round(2).replace(0.87, 17, inplace=True)\n",
    "df[\"a\"][df[\"a\"] < 4] = 19\n",
    "\n",
    "\"\"\"Conditionals and selectors\"\"\"\n",
    "df.loc[df[\"a\"] > 1, [\"a\",\"b\",\"target\"]]\n",
    "\n",
    "\"\"\"Selecting multiple column slices\"\"\"\n",
    "df.iloc[:, np.r_[0:2, 4:5]] \n",
    "\n",
    "\"\"\"apply and map examples\"\"\"\n",
    "df[[\"a\",\"b\",\"c\"]].applymap(lambda x: x+1)\n",
    "\n",
    "\"\"\"add 2 to row 3 and return the series\"\"\"\n",
    "df[[\"a\",\"b\",\"c\"]].apply(lambda x: x[0]+2,axis=0)\n",
    "\n",
    "\"\"\"add 3 to col A and return the series\"\"\"\n",
    "df.apply(lambda x: x['a']+1,axis=1)\n",
    "\n",
    "\"\"\" Split delimited values in a DataFrame column into two new columns \"\"\"\n",
    "df['new1'], df['new2'] = zip(*df['k'].apply(lambda x: x.split(': ', 1)))\n",
    "\n",
    "\"\"\" Doing calculations with DataFrame columns that have missing values\n",
    "  In example below, swap in 0 for df['col1'] cells that contain null \"\"\" \n",
    "df['new3'] = np.where(pd.isnull(df['b']),0,df['a']) + df['c']\n",
    "\n",
    "\"\"\" Exclude certain data type or include certain data types \"\"\"\n",
    "df.select_dtypes(exclude=['O','float'])\n",
    "df.select_dtypes(include=['int'])\n",
    "\n",
    "\"\"\"one liner to normalize a data frame\"\"\" \n",
    "(df[[\"a\",\"b\"]] - df[[\"a\",\"b\"]].mean()) / (df[[\"a\",\"b\"]].max() - df[[\"a\",\"b\"]].min())\n",
    "\n",
    "\"\"\"groupby used like a histogram to obtain counts on sub-ranges of a variable, pretty handy\"\"\" \n",
    "df.groupby(pd.cut(df.a, range(0, 1, 2))).size()\n",
    "\n",
    "\"\"\"use a local variable use inside a query of pandas using @\"\"\"\n",
    "mean = df[\"a\"].mean()\n",
    "df.query(\"a > @mean\")\n",
    "\n",
    "\"\"\"Calculate the % of missing values in each column\"\"\"\n",
    "df.isna().mean() \n",
    "\n",
    "\"\"\"Calculate the % of missing values in each row\"\"\"\n",
    "rows = df.isna().mean(axis=1) ; df.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kIRgQm5ikLyO"
   },
   "source": [
    "<a name=\"read-commands\"></a>\n",
    "\n",
    "**>>>  Read Commands**\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "id": "Spy9Vk4QmTcX",
    "outputId": "2b5cc70d-f548-4d12-e3e7-fd28fdb23355"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>foo1</td>\n",
       "      <td>2009-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>foo2</td>\n",
       "      <td>2009-01-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>foo3</td>\n",
       "      <td>2009-01-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>foo4</td>\n",
       "      <td>2009-01-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>foo5</td>\n",
       "      <td>2009-01-07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     A    B     C          D\n",
       "0  0.0  0.0  foo1 2009-01-01\n",
       "1  1.0  1.0  foo2 2009-01-02\n",
       "2  2.0  0.0  foo3 2009-01-05\n",
       "3  3.0  1.0  foo4 2009-01-06\n",
       "4  4.0  0.0  foo5 2009-01-07"
      ]
     },
     "execution_count": 17,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.util.testing.makeMixedDataFrame()\n",
    "df.to_csv(\"data.csv\") ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 238
    },
    "colab_type": "code",
    "id": "ggeKVjxKkQF-",
    "outputId": "434c0898-5bb7-4e31-ec66-7db66694f6bf"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>D</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2009-01-01</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>foo1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2009-01-02</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>foo2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2009-01-05</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>foo3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2009-01-06</th>\n",
       "      <td>3.0</td>\n",
       "      <td>1</td>\n",
       "      <td>foo4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2009-01-07</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>foo5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              A  B     C\n",
       "D                       \n",
       "2009-01-01  0.0  0  foo1\n",
       "2009-01-02  1.0  1  foo2\n",
       "2009-01-05  2.0  0  foo3\n",
       "2009-01-06  3.0  1  foo4\n",
       "2009-01-07  4.0  0  foo5"
      ]
     },
     "execution_count": 18,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"To avoid Unnamed: 0 when loading a previously saved csv with index\"\"\"\n",
    "\"\"\"To parse dates\"\"\"\n",
    "\"\"\"To set data types\"\"\"\n",
    "\n",
    "df_out = pd.read_csv(\"data.csv\", index_col=0,\n",
    "                 parse_dates=['D'],\n",
    "                 dtype={\"c\":\"category\", \"B\":\"int64\"}).set_index(\"D\")\n",
    "\n",
    "\"\"\"Copy data to clipboard; like an excel copy and paste\n",
    "df = pd.read_clipboard()\n",
    "\"\"\"\n",
    "\n",
    "\"\"\"Read table from website\n",
    "df = pd.read_html(url, match=\"table_name\")\n",
    "\"\"\"\n",
    "\n",
    "\"\"\" Read pdf into dataframe ()\n",
    "!pip install tabula\n",
    "from tabula import read_pdf\n",
    "df = read_pdf('test.pdf', pages='all')\n",
    "\"\"\"\n",
    "df_out.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WPA7xDSj2Zhy"
   },
   "source": [
    "<a name=\"create-ordered-categories\"></a>\n",
    "\n",
    "**>>> Create Ordered Categories**\n",
    "\n",
    "\n",
    "---\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "atBQQvlc26Er",
    "outputId": "d13e9eb4-ff97-4446-938f-34df5f8763f4"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>cats</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>bad</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>excellent</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target       cats\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227        bad\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994       good\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910  excellent"
      ]
     },
     "execution_count": 19,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df[\"cats\"] = [\"bad\",\"good\",\"excellent\"]; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 164
    },
    "colab_type": "code",
    "id": "6K0Sx67q213E",
    "outputId": "95d5336c-7340-4baf-dd5e-019b6dea88f1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Let's create our own categorical order.\n",
      "Now we can use logical sorting.\n",
      "We can also filter this as if they are numbers.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>cats</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>good</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>excellent</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target       cats\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994       good\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910  excellent"
      ]
     },
     "execution_count": 20,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from pandas.api.types import CategoricalDtype\n",
    "\n",
    "print(\"Let's create our own categorical order.\")\n",
    "cat_type = CategoricalDtype([\"bad\", \"good\", \"excellent\"], ordered = True)\n",
    "df[\"cats\"] = df[\"cats\"].astype(cat_type)\n",
    "\n",
    "print(\"Now we can use logical sorting.\")\n",
    "df = df.sort_values(\"cats\", ascending = True)\n",
    "\n",
    "print(\"We can also filter this as if they are numbers.\")\n",
    "df[df[\"cats\"] > \"bad\"]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KwvJXCalsc4C"
   },
   "source": [
    "<a name=\"select-columns-based-on-regex\"></a>\n",
    "\n",
    "**>>> Select Columns Based on Regex**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "9zH0yrN8snTH",
    "outputId": "18f57330-66ad-4c6a-a017-d2964f622780"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a_l</th>\n",
       "      <th>b_l</th>\n",
       "      <th>c_r</th>\n",
       "      <th>d_r</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      a_l     b_l     c_r     d_r  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 21,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.head(); df\n",
    "df.columns = [\"a_l\", \"b_l\", \"c_r\",\"d_r\",\"target\"]  ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "5K2mZhjBsl6W",
    "outputId": "1f376015-6399-45eb-a9e0-d0e08fc0f633"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a_l</th>\n",
       "      <th>b_l</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      a_l     b_l\n",
       "0  1.6243 -0.6118\n",
       "1  0.8654 -2.3015\n",
       "2  0.3190 -0.2494"
      ]
     },
     "execution_count": 22,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_out = df.filter(regex=\"_l\",axis=1) ; df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0gDw4jIqth2_"
   },
   "source": [
    "<a name=\"accessing-group-of-groupby-object\"></a>\n",
    "\n",
    "**>>> Accessing Group of Groupby Object**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 238
    },
    "colab_type": "code",
    "id": "lPOcCdettrbe",
    "outputId": "a6a9a24b-4478-453d-9c3f-afe424f7ff73"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>groupie</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>falcon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>hawk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>hawk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>eagle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>falcon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>hawk</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target groupie\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  falcon\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994    hawk\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910    hawk\n",
       "3  1.6243 -0.6118 -0.5282 -1.0730  1.1227   eagle\n",
       "4  0.8654 -2.3015  1.7448 -0.7612 -5.9994  falcon\n",
       "5  0.3190 -0.2494  1.4621 -2.0601 -0.5910    hawk"
      ]
     },
     "execution_count": 23,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df = df.append(df, ignore_index=True)\n",
    "df[\"groupie\"] = [\"falcon\",\"hawk\",\"hawk\",\"eagle\",\"falcon\",\"hawk\"]; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "id": "XB_7QRAgthGC",
    "outputId": "752b8842-e388-4e3a-e9c2-183a2c0768f8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "a         0.5012\n",
       "b        -0.9334\n",
       "c         1.5563\n",
       "d        -1.6272\n",
       "target   -2.3938\n",
       "dtype: float64"
      ]
     },
     "execution_count": 24,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbdf = df.groupby(\"groupie\")\n",
    "hawk = gbdf.get_group(\"hawk\").mean(); hawk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "we96Xwgfzz4o"
   },
   "source": [
    "<a name=\"multiple-external-selection-criteria\"></a>\n",
    "\n",
    "**>>> Multiple External Selection Criteria**\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AUcVNSPyz9fO"
   },
   "outputs": [],
   "source": [
    "df = df_test.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 81
    },
    "colab_type": "code",
    "id": "vcD86PKdz7Nd",
    "outputId": "3787a047-4b36-4232-f8cb-421916eca771"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994"
      ]
     },
     "execution_count": 26,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cr1 = df[\"a\"] > 0\n",
    "cr2 = df[\"b\"] < 0\n",
    "cr3 = df[\"c\"] > 0\n",
    "cr4 = df[\"d\"] >-1\n",
    "\n",
    "df[cr1 & cr2 & cr3 & cr4]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "h3M0lH1enaX0"
   },
   "source": [
    "<a name=\"memory-reduction-script\"></a>\n",
    "\n",
    "**>>> Memory Reduction Script (func)**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "2l8NYY0Snyvt",
    "outputId": "25ba776e-8502-45de-e663-51d32910c8ed"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 27,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 196
    },
    "colab_type": "code",
    "id": "dsOlKsjHnJA9",
    "outputId": "7d3adf1b-f8e8-453c-d493-4a1a25b4c8fb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 0.00 MB\n",
      "Memory usage after optimization is: 0.00 MB\n",
      "Decreased by 36.3%\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6240</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5283</td>\n",
       "      <td>-1.0732</td>\n",
       "      <td>1.1230</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8652</td>\n",
       "      <td>-2.3008</td>\n",
       "      <td>1.7451</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-6.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3191</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4619</td>\n",
       "      <td>-2.0605</td>\n",
       "      <td>-0.5908</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6240 -0.6118 -0.5283 -1.0732  1.1230\n",
       "1  0.8652 -2.3008  1.7451 -0.7612 -6.0000\n",
       "2  0.3191 -0.2494  1.4619 -2.0605 -0.5908"
      ]
     },
     "execution_count": 28,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gc\n",
    "\n",
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        gc.collect()\n",
    "        if col_type != object:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else:\n",
    "            df[col] = df[col].astype('category')\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df\n",
    "df_out = pv.reduce_mem_usage(df); df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9XAS5j-Orw7K"
   },
   "source": [
    "<a name=\"verify-primary-key\"></a>\n",
    "\n",
    "**>>> Verify Primary Key (func)**\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "7BU5Iix5r3U-",
    "outputId": "dd7f2481-8a9d-4336-9c0f-c35c2f48cbeb"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>first_d</th>\n",
       "      <th>second_d</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target  first_d  second_d\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227        0         4\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994        1         1\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910        2         9"
      ]
     },
     "execution_count": 29,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df[\"first_d\"] = [0,1,2]\n",
    "df[\"second_d\"] = [4,1,9] ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "dH7HG39CrweC",
    "outputId": "e7383bb6-fcb2-422b-dcce-70d0b6b3c5e0"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 31,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def verify_primary_key(df, column_list):\n",
    "    '''Verify if columns in column list can be treat as primary key'''\n",
    "\n",
    "    return df.shape[0] == df.groupby(column_list).size().reset_index().shape[0]\n",
    "\n",
    "verify_primary_key(df, [\"first_d\",\"second_d\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "t7rguQbH5lGZ"
   },
   "source": [
    "<a name=\"shift-columns-to-front\"></a>\n",
    "\n",
    "**>>> Shift Columns to Front (func)**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "vdHoOlpU5D_P",
    "outputId": "6eebaac9-7fe3-478a-d5fd-171057765428"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 32,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "GpK5aPCp4ocK",
    "outputId": "b732d7c8-0385-47e5-ff1c-68366f6df783"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.1227</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-5.9994</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.5910</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   target       c       d       a       b\n",
       "0  1.1227 -0.5282 -1.0730  1.6243 -0.6118\n",
       "1 -5.9994  1.7448 -0.7612  0.8654 -2.3015\n",
       "2 -0.5910  1.4621 -2.0601  0.3190 -0.2494"
      ]
     },
     "execution_count": 33,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def list_shuff(items, df):\n",
    "    \"Bring a list of columns to the front\"\n",
    "    cols = list(df)\n",
    "    for i in range(len(items)):\n",
    "        cols.insert(i, cols.pop(cols.index(items[i])))\n",
    "    df = df.loc[:, cols]\n",
    "    df.reset_index(drop=True, inplace=True)\n",
    "    return df\n",
    "\n",
    "df_out = pv.list_shuff([\"target\",\"c\",\"d\"],df); df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fUoY11rkqwOR"
   },
   "source": [
    "<a name=\"multiple-column-assignment\"></a>\n",
    "\n",
    "**>>> Multiple Column Assignments**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "pFT-wJ8crCnM",
    "outputId": "e81d9566-7cba-4e7d-964e-302fd3828701"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 34,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 100
    },
    "colab_type": "code",
    "id": "3ogBmCUTq0QA",
    "outputId": "5528c0f5-1ed3-4bb2-e721-4a7f93477b38"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "</style><table id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002\" ><caption>Average consumption</caption><thead>    <tr>        <th class=\"blank level0\" ></th>        <th class=\"col_heading level0 col0\" >a</th>        <th class=\"col_heading level0 col1\" >b</th>        <th class=\"col_heading level0 col2\" >c</th>        <th class=\"col_heading level0 col3\" >d</th>        <th class=\"col_heading level0 col4\" >target</th>        <th class=\"col_heading level0 col5\" >stringed</th>        <th class=\"col_heading level0 col6\" >ounces</th>        <th class=\"col_heading level0 col7\" >galons</th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row0_col0\" class=\"data row0 col0\" >1.624</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row0_col1\" class=\"data row0 col1\" >-0.6118</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row0_col2\" class=\"data row0 col2\" >-0.5282</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row0_col3\" class=\"data row0 col3\" >-1.073</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row0_col4\" class=\"data row0 col4\" >1.123</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row0_col5\" class=\"data row0 col5\" >1.6243453636632417</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row0_col6\" class=\"data row0 col6\" >-7.341</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row0_col7\" class=\"data row0 col7\" >0.01269</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002level0_row1\" class=\"row_heading level0 row1\" >2</th>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row1_col0\" class=\"data row1 col0\" >0.319</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row1_col1\" class=\"data row1 col1\" >-0.2494</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row1_col2\" class=\"data row1 col2\" >1.462</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row1_col3\" class=\"data row1 col3\" >-2.06</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row1_col4\" class=\"data row1 col4\" >-0.591</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row1_col5\" class=\"data row1 col5\" >0.31903909605709857</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row1_col6\" class=\"data row1 col6\" >-2.992</td>\n",
       "                        <td id=\"T_6d6167c4_627f_11ea_a8db_0242ac1c0002row1_col7\" class=\"data row1 col7\" >0.002492</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fba51e98908>"
      ]
     },
     "execution_count": 35,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_out = (df.assign(stringed = df[\"a\"].astype(str),\n",
    "            ounces = df[\"b\"]*12,#                                     this will allow yo set a title\n",
    "            galons = lambda df: df[\"a\"]/128)\n",
    "           .query(\"b > -1\")\n",
    "           .style.set_caption(\"Average consumption\")) ; df_out\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uRa-z96_sOJm"
   },
   "source": [
    "<a name=\"method-chaning-event\"></a>\n",
    "\n",
    "**>>> Method Chaining Technique**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "cWhfPOeCstaT",
    "outputId": "4282859b-56a3-4d16-dc5a-447ccd670c63"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0     NaN     NaN -0.5282     NaN     NaN\n",
       "1  0.8654 -2.3015     NaN     NaN -5.9994\n",
       "2  0.3190     NaN     NaN -2.0601     NaN"
      ]
     },
     "execution_count": 36,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df[df>df.mean()]  = None ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "pE8XM03TstVL",
    "outputId": "1c0cc4e8-ca1b-49b6-8440-da0d7cf76d06"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>b</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">0</th>\n",
       "      <th>-2.3</th>\n",
       "      <td>0.87</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>0.87</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           a    c    d  target\n",
       "  b                           \n",
       "0 -2.3  0.87  0.0  0.0    -6.0\n",
       "        0.87  0.0  0.0    -6.0"
      ]
     },
     "execution_count": 37,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# with line continuation character\n",
    "df_out = df.dropna(subset=[\"b\",\"c\"],how=\"all\") \\\n",
    ".loc[df[\"a\"]>0] \\\n",
    ".round(2) \\\n",
    ".groupby([\"target\",\"b\"]).max() \\\n",
    ".unstack() \\\n",
    ".fillna(0) \\\n",
    ".rolling(1).sum() \\\n",
    ".reset_index() \\\n",
    ".stack() \\\n",
    ".ffill().bfill() \n",
    "\n",
    "df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "i95nPbRQy-tY"
   },
   "source": [
    "<a name=\"load-multiple-files\"></a>\n",
    "\n",
    "**>>> Load Multiple Files**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iEpt2Lrvy9qQ"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.makedirs(\"folder\",exist_ok=True,); df_test.to_csv(\"folder/first.csv\",index=False) ; df_test.to_csv(\"folder/last.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HR_qwPhpx8y6"
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "files = glob.glob('folder/*.csv')\n",
    "dfs = [pd.read_csv(fp) for fp in files]\n",
    "df_out = pd.concat(dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 238
    },
    "colab_type": "code",
    "id": "oxiNknHKzqJy",
    "outputId": "7a6528b0-206d-4493-ac5e-732624b6cc31"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 40,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7F5IaBFm5Yr3"
   },
   "source": [
    "<a name=\"drop-rows-and-column-substring\"></a>\n",
    "\n",
    "**>>> Drop Rows with Column Substring**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "9MU0anIV5gwh",
    "outputId": "879a980d-d818-4367-be35-0170edccc06f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>string_feature</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>1xZoo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>Safe7x</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>bat4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target string_feature\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227          1xZoo\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994         Safe7x\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910           bat4"
      ]
     },
     "execution_count": 41,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df[\"string_feature\"] = [\"1xZoo\", \"Safe7x\", \"bat4\"]; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 112
    },
    "colab_type": "code",
    "id": "eCT7jaq15dty",
    "outputId": "d106838c-680a-4207-b1db-6f83641bbd79"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>string_feature</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>Safe7x</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>bat4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target string_feature\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994         Safe7x\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910           bat4"
      ]
     },
     "execution_count": 42,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "substring = [\"xZ\",\"7z\", \"tab4\"]\n",
    "\n",
    "df_out = df[~df.string_feature.str.contains('|'.join(substring))]; df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "D0QsjyL8Vpnx"
   },
   "source": [
    "<a name=\"explode-a-column\"></a>\n",
    "\n",
    "**>>> Unnest (Explode) a Column**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "aMYmKiiOVqH2",
    "outputId": "af3379df-3a91-4146-e50e-0e7107771715"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>g</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>[0a, 1a, 2a, 3a]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>[0b, 1b, 2b, 3b]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>[0c, 1c, 2c, 3c]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target                 g\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  [0a, 1a, 2a, 3a]\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994  [0b, 1b, 2b, 3b]\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910  [0c, 1c, 2c, 3c]"
      ]
     },
     "execution_count": 43,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.head()\n",
    "df[\"g\"] = [[str(a)+lista for a in range(4)] for lista in [\"a\",\"b\",\"c\"]]; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "id": "3KIcdqwKVy9A",
    "outputId": "abdb0975-21f1-4e74-d992-1008e83ea342"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>g</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>0a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>1a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>2a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>3a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>0b</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target   g\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  0a\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  1a\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  2a\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  3a\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994  0b"
      ]
     },
     "execution_count": 44,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_out = df.explode(\"g\"); df_out.iloc[:5,:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9euf-2Gq1V6x"
   },
   "source": [
    "<a name=\"nest-list-back-into-column\"></a>\n",
    "\n",
    "**>>> Nest List Back into Column**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7EIdSWIA1dke"
   },
   "outputs": [],
   "source": [
    "### Run above example first \n",
    "df = df_out.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "id": "hNvE6hOR1cm_",
    "outputId": "4b004dde-71d4-4c04-f740-302da5d804ab"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>g</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>[0a, 1a, 2a, 3a]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>[0a, 1a, 2a, 3a]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>[0a, 1a, 2a, 3a]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>[0a, 1a, 2a, 3a]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>[0b, 1b, 2b, 3b]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target                 g\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  [0a, 1a, 2a, 3a]\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  [0a, 1a, 2a, 3a]\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  [0a, 1a, 2a, 3a]\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  [0a, 1a, 2a, 3a]\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994  [0b, 1b, 2b, 3b]"
      ]
     },
     "execution_count": 46,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_out['g'] = df_out.groupby(df_out.index)['g'].agg(list); df_out.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nR35V2dLncry"
   },
   "source": [
    "<a name=\"split-cells-with-list\"></a>\n",
    "\n",
    "**>>> Split Cells With Lists**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "QZGQPt7nnjzq",
    "outputId": "1a70a056-6bed-41db-a790-ca1b215ead92"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>g</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>0a,1a,2a,3a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>0b,1b,2b,3b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>0c,1c,2c,3c</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target            g\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  0a,1a,2a,3a\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994  0b,1b,2b,3b\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910  0c,1c,2c,3c"
      ]
     },
     "execution_count": 47,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.head()\n",
    "df[\"g\"] = [\",\".join([str(a)+lista for a in range(4)]) for lista in [\"a\",\"b\",\"c\"]]; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "id": "eXPaKYbnnb52",
    "outputId": "d35b851a-4d6b-4dde-e87f-160d28b4be9e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>g</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>0a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>1a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>2a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>3a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>0b</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target   g\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  0a\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  1a\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  2a\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  3a\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994  0b"
      ]
     },
     "execution_count": 48,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_out = df.assign(g = df[\"g\"].str.split(\",\")).explode(\"g\"); df_out.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Yg8fCS_mEonR"
   },
   "source": [
    "<a name=\"table-exploration\"></a>\n",
    "\n",
    "### **Table Exploration**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HhMVuDosS6LV"
   },
   "source": [
    "<a name=\"groupby-functionality\"></a>\n",
    "\n",
    "**>>> Groupby Functionality**\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "MGg0GYSHSdI4",
    "outputId": "ef70b483-7129-4fcb-f56e-6874ec896240"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>gr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target  gr\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227   1\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994   1\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910   0"
      ]
     },
     "execution_count": 49,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.head() \n",
    "df[\"gr\"] = [1, 1 , 0] ;df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hF_X7hm1Sezm"
   },
   "outputs": [],
   "source": [
    "In [34]: gb.<TAB>  # noqa: E225, E999\n",
    "gb.agg        gb.boxplot    gb.cummin     gb.describe   gb.filter     \n",
    "gb.get_group  gb.height     gb.last       gb.median     gb.ngroups    \n",
    "gb.plot       gb.rank       gb.std        gb.transform  gb.aggregate  \n",
    "gb.count      gb.cumprod    gb.dtype      gb.first      gb.nth\n",
    "gb.groups     gb.hist       gb.max        gb.min        gb.gender        \n",
    "gb.prod       gb.resample   gb.sum        gb.var        gb.ohlc  \n",
    "gb.apply      gb.cummax     gb.cumsum     gb.fillna          \n",
    "gb.head       gb.indices    gb.mean       gb.name            \n",
    "gb.quantile   gb.size       gb.tail       gb.weight\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 175
    },
    "colab_type": "code",
    "id": "MuVeDT34TC_H",
    "outputId": "a66f6106-15e0-4895-f0aa-377f38337d42"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">a</th>\n",
       "      <th colspan=\"3\" halign=\"left\">b</th>\n",
       "      <th colspan=\"2\" halign=\"left\">c</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gr</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>0.3190</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>1.4621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.4898</td>\n",
       "      <td>1.2449</td>\n",
       "      <td>0.5367</td>\n",
       "      <td>-2.9133</td>\n",
       "      <td>-1.4566</td>\n",
       "      <td>1.1949</td>\n",
       "      <td>1.2166</td>\n",
       "      <td>0.6083</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         a                       b                       c        \n",
       "       sum    mean     std     sum    mean     std     sum    mean\n",
       "gr                                                                \n",
       "0   0.3190  0.3190     NaN -0.2494 -0.2494     NaN  1.4621  1.4621\n",
       "1   2.4898  1.2449  0.5367 -2.9133 -1.4566  1.1949  1.2166  0.6083"
      ]
     },
     "execution_count": 50,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_out = df.groupby('gr').agg([np.sum, np.mean, np.std]); df_out.iloc[:,:8]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CLSg3CgcKFI-"
   },
   "source": [
    "<a name=\"cross-correlation-series-without-duplicates\"></a>\n",
    "\n",
    "**>>> Cross Correlation Series Without Duplicates (func)**\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "oY4brAvfKGVC",
    "outputId": "6a6f9a76-f316-4e72-806d-450d390ac481"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 51,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 208
    },
    "colab_type": "code",
    "id": "yn8NeYwA421K",
    "outputId": "c9fbf8e4-0950-4748-f390-a41985a20196"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "b       target    0.9215\n",
       "a       d         0.6605\n",
       "        target    0.3206\n",
       "b       a        -0.0724\n",
       "c       d        -0.1764\n",
       "        b        -0.4545\n",
       "target  d        -0.4994\n",
       "c       target   -0.7647\n",
       "b       d        -0.7967\n",
       "a       c        -0.8555\n",
       "dtype: float64"
      ]
     },
     "execution_count": 52,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def corr_list(df):\n",
    "\n",
    "  return  (df.corr()\n",
    "          .unstack()\n",
    "          .sort_values(kind=\"quicksort\",ascending=False)\n",
    "          .drop_duplicates().iloc[1:]); df_out\n",
    "          \n",
    "pv.corr_list(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ALVc7KvFIu7A"
   },
   "source": [
    "<a name=\"missing-data-report\"></a>\n",
    "\n",
    "**>>> Missing Data Report (func)**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "bsVEVy3kIoO3",
    "outputId": "9cb91c58-cf5c-4e52-b24f-e8df9d1e5465"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0     NaN     NaN -0.5282     NaN     NaN\n",
       "1  0.8654 -2.3015     NaN     NaN -5.9994\n",
       "2  0.3190     NaN     NaN -2.0601     NaN"
      ]
     },
     "execution_count": 53,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df[df>df.mean()]  = None ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "colab_type": "code",
    "id": "33KlxZT2IjyC",
    "outputId": "3b477e40-dfbd-4580-c8fe-068606b71a2a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">a</th>\n",
       "      <th colspan=\"3\" halign=\"left\">b</th>\n",
       "      <th colspan=\"3\" halign=\"left\">c</th>\n",
       "      <th colspan=\"3\" halign=\"left\">d</th>\n",
       "      <th colspan=\"3\" halign=\"left\">target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>sum</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gr</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>0.3190</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.4898</td>\n",
       "      <td>1.2449</td>\n",
       "      <td>0.5367</td>\n",
       "      <td>-2.9133</td>\n",
       "      <td>-1.4566</td>\n",
       "      <td>1.1949</td>\n",
       "      <td>1.2166</td>\n",
       "      <td>0.6083</td>\n",
       "      <td>1.6072</td>\n",
       "      <td>-1.8342</td>\n",
       "      <td>-0.9171</td>\n",
       "      <td>0.2204</td>\n",
       "      <td>-4.8767</td>\n",
       "      <td>-2.4384</td>\n",
       "      <td>5.0361</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         a                       b                       c                       d                  target                \n",
       "       sum    mean     std     sum    mean     std     sum    mean     std     sum    mean     std     sum    mean     std\n",
       "gr                                                                                                                        \n",
       "0   0.3190  0.3190     NaN -0.2494 -0.2494     NaN  1.4621  1.4621     NaN -2.0601 -2.0601     NaN -0.5910 -0.5910     NaN\n",
       "1   2.4898  1.2449  0.5367 -2.9133 -1.4566  1.1949  1.2166  0.6083  1.6072 -1.8342 -0.9171  0.2204 -4.8767 -2.4384  5.0361"
      ]
     },
     "execution_count": 54,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "def missing_data(data):\n",
    "    \"Create a dataframe with a percentage and count of missing values\"\n",
    "    total = data.isnull().sum().sort_values(ascending = False)\n",
    "    percent = (data.isnull().sum()/data.isnull().count()*100).sort_values(ascending = False)\n",
    "    return pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n",
    "\n",
    "pv.df_out = missing_data(df); df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Vx3L31ona_Hz"
   },
   "source": [
    "<a name=\"duplicated-rows-report\"></a>\n",
    "\n",
    "**>>> Duplicated Rows Report**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "SXyLCYzubGm2",
    "outputId": "9e08c57e-d9d8-46b9-cee9-59567cb66eb5"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.8654 -2.3015  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 55,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df[\"a\"].iloc[2] = df[\"a\"].iloc[1]\n",
    "df[\"b\"].iloc[2] = df[\"b\"].iloc[1] ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 112
    },
    "colab_type": "code",
    "id": "AlUzCevDRrlA",
    "outputId": "ef7d4d0d-5a87-40e1-b604-c5f3b770967c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.8654 -2.3015  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 56,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get a report of all duplicate records in a dataframe, based on specific columns\n",
    "df_out = df[df.duplicated(['a', 'b'], keep=False)] ; df_out\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0S6_tOl3tvAB"
   },
   "source": [
    "<a name=\"skewness\"></a>\n",
    "\n",
    "**>>> Skewness (func)**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "lOxUEe7xt0el",
    "outputId": "2f403056-03a3-4a5a-f189-86a24d6cd2ac"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 57,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "id": "ZG6u2rtZtwp2",
    "outputId": "c7ad814f-57f7-4416-b08f-065ca9477a0e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>var_name</th>\n",
       "      <th>col_type</th>\n",
       "      <th>skew_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a</td>\n",
       "      <td>float64</td>\n",
       "      <td>0.1963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b</td>\n",
       "      <td>float64</td>\n",
       "      <td>-0.6210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c</td>\n",
       "      <td>float64</td>\n",
       "      <td>-0.6659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>d</td>\n",
       "      <td>float64</td>\n",
       "      <td>-0.5427</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>target</td>\n",
       "      <td>float64</td>\n",
       "      <td>-0.5418</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  var_name col_type  skew_value\n",
       "0        a  float64      0.1963\n",
       "1        b  float64     -0.6210\n",
       "2        c  float64     -0.6659\n",
       "3        d  float64     -0.5427\n",
       "4   target  float64     -0.5418"
      ]
     },
     "execution_count": 59,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.stats import skew\n",
    "\n",
    "def display_skewness(data):\n",
    "    '''show skewness information\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        data: pandas dataframe\n",
    "\n",
    "        Return\n",
    "        ------\n",
    "        df: pandas dataframe\n",
    "    '''\n",
    "    numeric_cols = data.columns[data.dtypes != 'object'].tolist()\n",
    "    skew_value = []\n",
    "\n",
    "    for i in numeric_cols:\n",
    "        skew_value += [skew(data[i])]\n",
    "    df = pd.concat(\n",
    "        [pd.Series(numeric_cols), pd.Series(data.dtypes[data.dtypes != 'object'].apply(lambda x: str(x)).values)\n",
    "            , pd.Series(skew_value)], axis=1)\n",
    "    df.columns = ['var_name', 'col_type', 'skew_value']\n",
    "\n",
    "    return df\n",
    "\n",
    "display_skewness(df)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iRr6mnrtiEWT"
   },
   "source": [
    "<a name=\"feature-processing\"></a>\n",
    "\n",
    "### **Feature Processing**\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "z7xZVDLA1360"
   },
   "source": [
    "<a name=\"remove-correlated-pairs\"></a>\n",
    "\n",
    "**>>> Remove Correlated Pairs (func)**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "f1QnY6I4tt8F",
    "outputId": "ce4d42c5-646a-4aaf-a9d0-f0e2a31f797c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 60,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df= df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "3wIMhjKis4Sz",
    "outputId": "14e0fc37-83dd-4782-9de3-2991ea1603fe"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b  target\n",
       "0  1.6243 -0.6118  1.1227\n",
       "1  0.8654 -2.3015 -5.9994\n",
       "2  0.3190 -0.2494 -0.5910"
      ]
     },
     "execution_count": 61,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def drop_corr(df, thresh=0.99,keep_cols=[]):\n",
    "    df_corr = df.corr().abs()\n",
    "    upper = df_corr.where(np.triu(np.ones(df_corr.shape), k=1).astype(np.bool))\n",
    "    to_remove = [column for column in upper.columns if any(upper[column] > thresh)] ## Change to 99% for selection\n",
    "    to_remove = [x for x in to_remove if x not in keep_cols]\n",
    "    df_corr = df_corr.drop(columns = to_remove)\n",
    "    return df.drop(to_remove,axis=1)\n",
    "\n",
    "df_out = pv.drop_corr(df, thresh=0.1,keep_cols=[\"target\"]); df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "a1qx0vuIkinF"
   },
   "source": [
    "<a name=\"replace-infrequently-occuring-categories\"></a>\n",
    "\n",
    "**>>> Replace Infrequently Occuring Categories**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 332
    },
    "colab_type": "code",
    "id": "i2PCRT9lkyqW",
    "outputId": "2e17e202-d2e7-4d46-9de9-836958690fad"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>bat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>bat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>rat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>mat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>mat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>mat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>mat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>mat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>mat</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target  cat\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  bat\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994  bat\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910  rat\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  mat\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994  mat\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910  mat\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  mat\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994  mat\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910  mat"
      ]
     },
     "execution_count": 62,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df = df.append([df]*2)\n",
    "df[\"cat\"] = [\"bat\",\"bat\",\"rat\",\"mat\",\"mat\",\"mat\",\"mat\",\"mat\",\"mat\"]; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "id": "0bC6_yp_ks7E",
    "outputId": "06a59535-fe6a-412f-ba92-f9b78bcb97df"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>cat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>bat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>bat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>Other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>mat</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>mat</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target    cat\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227    bat\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994    bat\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910  Other\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227    mat\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994    mat"
      ]
     },
     "execution_count": 63,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "def replace_small_cat(df, columns, thresh=0.2, term=\"other\"):\n",
    "  for col in columns:\n",
    "\n",
    "    # Step 1: count the frequencies\n",
    "    frequencies = df[col].value_counts(normalize = True)\n",
    "\n",
    "  # Step 2: establish your threshold and filter the smaller categories\n",
    "\n",
    "    small_categories = frequencies[frequencies < thresh].index\n",
    "\n",
    "    df[col] = df[col].replace(small_categories, \"Other\")\n",
    "    \n",
    "  return df\n",
    "\n",
    "df_out = pv.replace_small_cat(df,[\"cat\"]); df_out.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "r39IzmYViz4y"
   },
   "source": [
    "<a name=\"quasi-constant-feature-detection\"></a>\n",
    "\n",
    "**>>> Quasi-Constant Features Detection (func)**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "i4QkTO5ni72l"
   },
   "outputs": [],
   "source": [
    "df = df_test.copy()\n",
    "df[\"a\"] = 3 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 161
    },
    "colab_type": "code",
    "id": "331D2iDMKxdl",
    "outputId": "42b3bfdf-c2c4-4eb2-d736-b9085fbcc35c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1  variables are found to be almost constant\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        b       c       d  target\n",
       "0 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 65,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "def constant_feature_detect(data,threshold=0.98):\n",
    "    \"\"\" detect features that show the same value for the \n",
    "    majority/all of the observations (constant/quasi-constant features)\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    data : pd.Dataframe\n",
    "    threshold : threshold to identify the variable as constant\n",
    "        \n",
    "    Returns\n",
    "    -------\n",
    "    list of variables names\n",
    "    \"\"\"\n",
    "    \n",
    "    data_copy = data.copy(deep=True)\n",
    "    quasi_constant_feature = []\n",
    "    for feature in data_copy.columns:\n",
    "        predominant = (data_copy[feature].value_counts() / np.float(\n",
    "                      len(data_copy))).sort_values(ascending=False).values[0]\n",
    "        if predominant >= threshold:\n",
    "            quasi_constant_feature.append(feature)\n",
    "    print(len(quasi_constant_feature),' variables are found to be almost constant')    \n",
    "    return quasi_constant_feature\n",
    "\n",
    "# the original dataset has no constant variable\n",
    "qconstant_col = pv.constant_feature_detect(data=df,threshold=0.9)\n",
    "df_out = df.drop(qconstant_col, axis=1) ; df_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "711ozFdIkSZd"
   },
   "outputs": [],
   "source": [
    "### I will take care of outliers separately"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FsZ9OD17eJjQ"
   },
   "source": [
    "<a name=\"filling-missing-values-separately\"></a>\n",
    "\n",
    "**>>> Filling Missing Values Separately**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "k_wVYMJCeXtg",
    "outputId": "8faa52cc-af2d-4671-f374-e3b843cb2e11"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0     NaN     NaN -0.5282     NaN     NaN\n",
       "1  0.8654 -2.3015     NaN     NaN -5.9994\n",
       "2  0.3190     NaN     NaN -2.0601     NaN"
      ]
     },
     "execution_count": 67,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df[df>df.mean()]  = None ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "6u2GC_u-eOxk",
    "outputId": "b8087aa8-b4a2-4389-92f8-71ca8c67b6b0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.0000</td>\n",
       "      <td>3.0000</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>9999.0000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>5.0000</td>\n",
       "      <td>9999.0000</td>\n",
       "      <td>-5.999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>3.0000</td>\n",
       "      <td>5.0000</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c          d target\n",
       "0  4.0000  3.0000 -0.5282  9999.0000  False\n",
       "1  0.8654 -2.3015  5.0000  9999.0000 -5.999\n",
       "2  0.3190  3.0000  5.0000    -2.0601  False"
      ]
     },
     "execution_count": 68,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Clean up missing values in multiple DataFrame columns\n",
    "dict_fill = {'a': 4,\n",
    "              'b': 3,\n",
    "              'c': 5,\n",
    "              'd': 9999,\n",
    "              'target': \"False\"}\n",
    "df = df.fillna(dict_fill) ;df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dPA4QjGVf-7p"
   },
   "source": [
    "<a name=\"conditioned-column-value-replacement\"></a>\n",
    "\n",
    "**>>> Conditioned Column Value Replacement**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "3vLPzcxzgWI4",
    "outputId": "185844ec-7662-4d37-8136-5d56c79910ff"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 69,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "1R9DJLR0gPs3",
    "outputId": "5e291047-ae48-4a58-880a-2f70d4f8e306"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730     NaN\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 70,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Set DataFrame column values based on other column values\n",
    "df.loc[(df['a'] >1 ) & (df['c'] <0), ['target']] = np.nan ;df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EIRe1HJ9hE5M"
   },
   "source": [
    "<a name=\"remove-non-numeric-values-in-data-frame\"></a>\n",
    "\n",
    "**>>> Remove Non-numeric Values in Data Frame**\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "FaGb-96FhQa2",
    "outputId": "2c52702b-28be-4e5d-ed2e-a9e83e592a4e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TI4560L1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.6243SC-0.6118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TI4560L0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>0.8654SC-2.3015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TI4560L0.319</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>0.319SC-0.2494</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               a       b       c       d           target\n",
       "0  TI4560L1.6243 -0.6118 -0.5282 -1.0730  1.6243SC-0.6118\n",
       "1  TI4560L0.8654 -2.3015  1.7448 -0.7612  0.8654SC-2.3015\n",
       "2   TI4560L0.319 -0.2494  1.4621 -2.0601   0.319SC-0.2494"
      ]
     },
     "execution_count": 71,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy().assign(target=lambda row: row[\"a\"].round(4).astype(str)+\"SC\"+row[\"b\"].round(4).astype(str))\n",
    "df[\"a\"] = \"TI4560L\" + df[\"a\"].round(4).astype(str) ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "Hj_S3ZglhQXE",
    "outputId": "1fb5b1cc-94a1-4d83-ba52-adbe3149099a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>456016243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1624306118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>456008654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>0865423015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>45600319</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>031902494</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           a       b       c       d      target\n",
       "0  456016243 -0.6118 -0.5282 -1.0730  1624306118\n",
       "1  456008654 -2.3015  1.7448 -0.7612  0865423015\n",
       "2   45600319 -0.2494  1.4621 -2.0601   031902494"
      ]
     },
     "execution_count": 72,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_out = df.replace('[^0-9]+', '', regex=True); df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nt_XKRfeubGR"
   },
   "source": [
    "<a name=\"feature-scaling-normalisation-standardisation\"></a>\n",
    "\n",
    "**>>> Feature Scaling, Normalisation, Standardisation (func)**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "BpJGV3ivuito",
    "outputId": "5e7c5b9f-7f3d-4389-906b-970ffa704c1d"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 73,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df= df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "JufFh9G6uiW5",
    "outputId": "c615ab55-e148-4e8d-afd0-dea50dc35cdb"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>0.8234</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.76</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.8756</td>\n",
       "      <td>0.00</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c     d  target\n",
       "0  1.6243  0.8234  0.0000  0.76  1.1227\n",
       "1  0.8654  0.0000  1.0000  1.00 -5.9994\n",
       "2  0.3190  1.0000  0.8756  0.00 -0.5910"
      ]
     },
     "execution_count": 74,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "def scaler(df,scaler=None,train=True, target=None, cols_ignore=None, type=\"Standard\"):\n",
    "\n",
    "  if cols_ignore:\n",
    "    hold = df[cols_ignore].copy()\n",
    "    df = df.drop(cols_ignore,axis=1)\n",
    "  if target:\n",
    "    x = df.drop([target],axis=1).values #returns a numpy array\n",
    "  else:\n",
    "    x = df.values\n",
    "  if train:\n",
    "    if type==\"Standard\":\n",
    "      scal = StandardScaler()\n",
    "    elif type==\"MinMax\":\n",
    "      scal = MinMaxScaler()\n",
    "    scal.fit(x)\n",
    "    x_scaled = scal.transform(x)\n",
    "  else:\n",
    "    x_scaled = scaler.transform(x)\n",
    "  \n",
    "  if target:\n",
    "    df_out = pd.DataFrame(x_scaled, index=df.index, columns=df.drop([target],axis=1).columns)\n",
    "    df_out[target]= df[target]\n",
    "  else:\n",
    "    df_out = pd.DataFrame(x_scaled, index=df.index, columns=df.columns)\n",
    "  \n",
    "  df_out = pd.concat((hold,df_out),axis=1)\n",
    "  if train:\n",
    "    return df_out, scal\n",
    "  else:\n",
    "    return df_out\n",
    "\n",
    "df_out_train, scl = pv.scaler(df,target=\"target\",cols_ignore=[\"a\"],type=\"MinMax\")\n",
    "df_out_test = pv.scaler(df_test,scaler=scl,train=False, target=\"target\",cols_ignore=[\"a\"]); df_out_test\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0l5tGRIDvTrP"
   },
   "source": [
    "<a name=\"impute-null-with-tail-distribution\"></a>\n",
    "\n",
    "**>>> Impute Null with Tail Distribution (func)**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "9W_fxg6vvx_8",
    "outputId": "382c97be-cbf9-44ea-aabc-1dda281a563f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0     NaN     NaN -0.5282     NaN     NaN\n",
       "1  0.8654 -2.3015     NaN     NaN -5.9994\n",
       "2  0.3190     NaN     NaN -2.0601     NaN"
      ]
     },
     "execution_count": 75,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df[df>df.mean()]  = None ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "BI_eRvyAvOc9",
    "outputId": "dac9e8eb-3db4-4fc2-f274-8dea7ef03053"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.7512</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.7512     NaN -0.5282     NaN     NaN\n",
       "1  0.8654 -2.3015     NaN     NaN -5.9994\n",
       "2  0.3190     NaN     NaN -2.0601     NaN"
      ]
     },
     "execution_count": 76,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def impute_null_with_tail(df,cols=[]):\n",
    "    \"\"\"\n",
    "    replacing the NA by values that are at the far end of the distribution of that variable\n",
    "    calculated by mean + 3*std\n",
    "    \"\"\"\n",
    "    \n",
    "    df = df.copy(deep=True)\n",
    "    for i in cols:\n",
    "        if df[i].isnull().sum()>0:\n",
    "            df[i] = df[i].fillna(df[i].mean()+3*df[i].std())\n",
    "        else:\n",
    "            warn(\"Column %s has no missing\" % i)\n",
    "    return df    \n",
    "  \n",
    "\n",
    "df_out = pv.impute_null_with_tail(df,cols=df.columns); df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Ss0OE9lwu2jE"
   },
   "source": [
    "<a name=\"detect-outliers\"></a>\n",
    "\n",
    "**>>> Detect Outliers (func)**\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "E0yIbe-myI40",
    "outputId": "ed849563-928f-41ad-ef7f-850462fda034"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 77,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 86
    },
    "colab_type": "code",
    "id": "09aL40KKwiqF",
    "outputId": "e650ca7b-b744-4829-8b5a-9a8c70bfa725"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num of outlier detected: 1\n",
      "Proportion of outlier detected 0.3333333333333333\n",
      "Upper bound: 1.5712030633954956 \n",
      "Lower bound: 0.2658967957893529\n"
     ]
    }
   ],
   "source": [
    "def outlier_detect(data,col,threshold=3,method=\"IQR\"):\n",
    "  \n",
    "    if method == \"IQR\":\n",
    "      IQR = data[col].quantile(0.75) - data[col].quantile(0.25)\n",
    "      Lower_fence = data[col].quantile(0.25) - (IQR * threshold)\n",
    "      Upper_fence = data[col].quantile(0.75) + (IQR * threshold)\n",
    "    if method == \"STD\":\n",
    "      Upper_fence = data[col].mean() + threshold * data[col].std()\n",
    "      Lower_fence = data[col].mean() - threshold * data[col].std()   \n",
    "    if method == \"OWN\":\n",
    "      Upper_fence = data[col].mean() + threshold * data[col].std()\n",
    "      Lower_fence = data[col].mean() - threshold * data[col].std() \n",
    "    if method ==\"MAD\":\n",
    "      median = data[col].median()\n",
    "      median_absolute_deviation = np.median([np.abs(y - median) for y in data[col]])\n",
    "      modified_z_scores = pd.Series([0.6745 * (y - median) / median_absolute_deviation for y in data[col]])\n",
    "      outlier_index = np.abs(modified_z_scores) > threshold\n",
    "      print('Num of outlier detected:',outlier_index.value_counts()[1])\n",
    "      print('Proportion of outlier detected',outlier_index.value_counts()[1]/len(outlier_index))\n",
    "      return outlier_index, (median_absolute_deviation, median_absolute_deviation)\n",
    "\n",
    "\n",
    "    para = (Upper_fence, Lower_fence)\n",
    "    tmp = pd.concat([data[col]>Upper_fence,data[col]<Lower_fence],axis=1)\n",
    "    outlier_index = tmp.any(axis=1)\n",
    "    print('Num of outlier detected:',outlier_index.value_counts()[1])\n",
    "    print('Proportion of outlier detected',outlier_index.value_counts()[1]/len(outlier_index))\n",
    "    \n",
    "    return outlier_index, para\n",
    "    \n",
    "\n",
    "index,para = pv.outlier_detect(df,\"a\",threshold=0.5,method=\"IQR\")\n",
    "print('Upper bound:',para[0],'\\nLower bound:',para[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "h0-FX0ofzb_5"
   },
   "source": [
    "<a name=\"windsorize-outliers\"></a>\n",
    "\n",
    "**>>> Windsorize Outliers (func)**\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "aP4tFcchygi-",
    "outputId": "89e8c38a-183a-47fc-b4f9-a0427a6cc93d"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 79,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# RUN above example first\n",
    "df = df_test.copy(); df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "QX3kh1wkyd7w",
    "outputId": "900be932-f35a-429f-f26d-c5ee078cadc1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5712</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.5712 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 80,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def windsorization(data,col,para,strategy='both'):\n",
    "    \"\"\"\n",
    "    top-coding & bottom coding (capping the maximum of a distribution at an arbitrarily set value,vice versa)\n",
    "    \"\"\"\n",
    "\n",
    "    data_copy = data.copy(deep=True)  \n",
    "    if strategy == 'both':\n",
    "        data_copy.loc[data_copy[col]>para[0],col] = para[0]\n",
    "        data_copy.loc[data_copy[col]<para[1],col] = para[1]\n",
    "    elif strategy == 'top':\n",
    "        data_copy.loc[data_copy[col]>para[0],col] = para[0]\n",
    "    elif strategy == 'bottom':\n",
    "        data_copy.loc[data_copy[col]<para[1],col] = para[1]  \n",
    "    return data_copy\n",
    "\n",
    "\n",
    "df_out = pv.windsorization(data=df,col='a',para=para,strategy='both'); df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Sj9tHDyy0yrX"
   },
   "source": [
    "<a name=\"drop-outiers\"></a>\n",
    "\n",
    "**>>> Drop Outliers**\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "KCBy92lH1CRQ",
    "outputId": "b9e4dbab-233b-43aa-adc0-080ca1fab72c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 81,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## run the top two examples\n",
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 112
    },
    "colab_type": "code",
    "id": "s-NJN_pQ0926",
    "outputId": "7e4d89a2-dc9f-4a53-889f-98fd3ca3320b"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 82,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_out = df[~index] ; df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "x9PY8t5_1Swl"
   },
   "source": [
    "<a name=\"impute-outiers (func)\"></a>\n",
    "\n",
    "**>>> Impute Outliers**\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "Tdbz7JGd1TXb",
    "outputId": "75dfbddc-a580-4f3f-efd2-753298148642"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.9363</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  0.9363 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 83,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def impute_outlier(data,col,outlier_index,strategy='mean'):\n",
    "    \"\"\"\n",
    "    impute outlier with mean/median/most frequent values of that variable.\n",
    "    \"\"\"\n",
    "\n",
    "    data_copy = data.copy(deep=True)\n",
    "    if strategy=='mean':\n",
    "        data_copy.loc[outlier_index,col] = data_copy[col].mean()\n",
    "    elif strategy=='median':\n",
    "        data_copy.loc[outlier_index,col] = data_copy[col].median()\n",
    "    elif strategy=='mode':\n",
    "        data_copy.loc[outlier_index,col] = data_copy[col].mode()[0]   \n",
    "        \n",
    "    return data_copy\n",
    "  \n",
    "df_out = pv.impute_outlier(data=df,col='a', outlier_index=index,strategy='mean'); df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "43TXWKLvkjoE"
   },
   "source": [
    "<a name=\"feature-engineering\"></a>\n",
    "\n",
    "### **Feature Engineering**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aWoEL8dvk5xf"
   },
   "source": [
    "<a name=\"automate-dummy-encodings\"></a>\n",
    "\n",
    "**>>> Automated Dummy (one-hot) Encoding (func)**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yiXoHGn3lDC6"
   },
   "outputs": [],
   "source": [
    "df = df_test.copy()\n",
    "df[\"e\"] = np.where(df[\"c\"]> df[\"a\"], 1,  2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 161
    },
    "colab_type": "code",
    "id": "i-x5XAehjV83",
    "outputId": "1a5401c1-6245-404e-e24e-9734da0b268b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "e\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>e_1</th>\n",
       "      <th>e_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target  e_1  e_2\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227    0    1\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994    1    0\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910    1    0"
      ]
     },
     "execution_count": 85,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def auto_dummy(df, unique=15):\n",
    "  # Creating dummies for small object uniques\n",
    "  if len(df)<unique:\n",
    "    raise ValueError('unique is set higher than data lenght')\n",
    "  list_dummies =[]\n",
    "  for col in df.columns:\n",
    "      if (len(df[col].unique()) < unique):\n",
    "          list_dummies.append(col)\n",
    "          print(col)\n",
    "  df_edit = pd.get_dummies(df, columns = list_dummies) # Saves original dataframe\n",
    "  #df_edit = pd.concat([df[[\"year\",\"qtr\"]],df_edit],axis=1)\n",
    "  return df_edit\n",
    "\n",
    "df_out = pv.auto_dummy(df, unique=3); df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SPI9e4mzm__u"
   },
   "source": [
    "<a name=\"binarise-empty-columns\"></a>\n",
    "\n",
    "**>>> Binarise Empty Columns (func)**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "AfzrUztOnr_D",
    "outputId": "777a25bc-ad4d-4bbb-dcb4-16f7b9b97674"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0     NaN     NaN -0.5282     NaN     NaN\n",
       "1  0.8654 -2.3015     NaN     NaN -5.9994\n",
       "2  0.3190     NaN     NaN -2.0601     NaN"
      ]
     },
     "execution_count": 86,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df[df>df.mean()]  = None ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 213
    },
    "colab_type": "code",
    "id": "ID9Hq6-3l9P3",
    "outputId": "b2bd76cf-7810-4aec-d35a-40caf39c45d8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "b\n",
      "c\n",
      "d\n",
      "target\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b_0</th>\n",
       "      <th>b_1</th>\n",
       "      <th>c_0</th>\n",
       "      <th>c_1</th>\n",
       "      <th>d_0</th>\n",
       "      <th>d_1</th>\n",
       "      <th>target_0</th>\n",
       "      <th>target_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a  b_0  b_1  c_0  c_1  d_0  d_1  target_0  target_1\n",
       "0     NaN    1    0    0    1    1    0         1         0\n",
       "1  0.8654    0    1    1    0    1    0         0         1\n",
       "2  0.3190    1    0    1    0    0    1         1         0"
      ]
     },
     "execution_count": 87,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def binarise_empty(df, frac=80):\n",
    "  # Binarise slightly empty columns\n",
    "  this =[]\n",
    "  for col in df.columns:\n",
    "      if df[col].dtype != \"object\":\n",
    "          is_null = df[col].isnull().astype(int).sum()\n",
    "          if (is_null/df.shape[0]) >frac: # if more than 70% is null binarise\n",
    "              print(col)\n",
    "              this.append(col)\n",
    "              df[col] = df[col].astype(float)\n",
    "              df[col] = df[col].apply(lambda x: 0 if (np.isnan(x)) else 1)\n",
    "  df = pd.get_dummies(df, columns = this) \n",
    "  return df\n",
    "\n",
    "df_out = pv.binarise_empty(df, frac=0.6); df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lvmZzT0IohzV"
   },
   "source": [
    "<a name=\"polynomials\"></a>\n",
    "\n",
    "**>>> Polynomials (func)**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "XFMboTB1q08S",
    "outputId": "a6296e0a-e875-452d-8e87-8d2ab3766a15"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 88,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "L8Y7gFRdoMLb",
    "outputId": "f77f5226-8951-4126-89d7-ea7372bb4f5c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>a/b</th>\n",
       "      <th>aXb</th>\n",
       "      <th>b/a</th>\n",
       "      <th>bXa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>0.9613</td>\n",
       "      <td>-0.9937</td>\n",
       "      <td>-0.4687</td>\n",
       "      <td>-0.9937</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>inf</td>\n",
       "      <td>-1.9918</td>\n",
       "      <td>-4.2124</td>\n",
       "      <td>-1.9918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>0.1555</td>\n",
       "      <td>-0.0796</td>\n",
       "      <td>-inf</td>\n",
       "      <td>-0.0796</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target     a/b     aXb     b/a     bXa\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  0.9613 -0.9937 -0.4687 -0.9937\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994     inf -1.9918 -4.2124 -1.9918\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910  0.1555 -0.0796    -inf -0.0796"
      ]
     },
     "execution_count": 89,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def polynomials(df, feature_list):\n",
    "  for feat in feature_list:\n",
    "    for feat_two in feature_list:\n",
    "      if feat==feat_two:\n",
    "        continue\n",
    "      else:\n",
    "       df[feat+\"/\"+feat_two] = df[feat]/(df[feat_two]-df[feat_two].min()) #zero division guard\n",
    "       df[feat+\"X\"+feat_two] = df[feat]*(df[feat_two])\n",
    "\n",
    "  return df\n",
    "\n",
    "df_out = pv.polynomials(df, [\"a\",\"b\"]) ; df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "IN7fF1gz_xnv"
   },
   "source": [
    "<a name=\"transformations\"></a>\n",
    "\n",
    "**>>> Transformations (func)**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "hbtI8eceAlFt",
    "outputId": "1f0772dc-52c2-45c6-c41d-cec92c70e6b3"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 90,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "GkKXeNHQ_534",
    "outputId": "497c2ca6-0676-4bb5-c1ae-426bf2a32b51"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>a_POWER_2</th>\n",
       "      <th>b_POWER_2</th>\n",
       "      <th>a_LOG_p_one_abs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>1.7038</td>\n",
       "      <td>2.8554</td>\n",
       "      <td>0.8352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>0.2985</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.4359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>4.2114</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target  a_POWER_2  b_POWER_2  a_LOG_p_one_abs\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227     1.7038     2.8554           0.8352\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994     0.2985     0.0000           0.4359\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910     0.0000     4.2114           0.0000"
      ]
     },
     "execution_count": 91,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def transformations(df,features):\n",
    "  df_new = df[features]\n",
    "  df_new = df_new - df_new.min()\n",
    "\n",
    "  sqr_name = [str(fa)+\"_POWER_2\" for fa in df_new.columns]\n",
    "  log_p_name = [str(fa)+\"_LOG_p_one_abs\" for fa in df_new.columns]\n",
    "  rec_p_name = [str(fa)+\"_RECIP_p_one\" for fa in df_new.columns]\n",
    "  sqrt_name = [str(fa)+\"_SQRT_p_one\" for fa in df_new.columns]\n",
    "\n",
    "  df_sqr = pd.DataFrame(np.power(df_new.values, 2),columns=sqr_name, index=df.index)\n",
    "  df_log = pd.DataFrame(np.log(df_new.add(1).abs().values),columns=log_p_name, index=df.index)\n",
    "  df_rec = pd.DataFrame(np.reciprocal(df_new.add(1).values),columns=rec_p_name, index=df.index)\n",
    "  df_sqrt = pd.DataFrame(np.sqrt(df_new.abs().add(1).values),columns=sqrt_name, index=df.index)\n",
    "\n",
    "  dfs = [df, df_sqr, df_log, df_rec, df_sqrt]\n",
    "\n",
    "  df=  pd.concat(dfs, axis=1)\n",
    "\n",
    "  return df\n",
    "\n",
    "df_out = pv.transformations(df,[\"a\",\"b\"]); df_out.iloc[:,:8]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YOFkPHmBw5RF"
   },
   "source": [
    "<a name=\"genetic-programming\"></a>\n",
    "\n",
    "**>>> Genetic Programming**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 193
    },
    "colab_type": "code",
    "id": "GvTSl5--tmhH",
    "outputId": "4dd97860-a58d-4b44-d013-b4c105eef74b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting gplearn\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/43/6b/ee38cd74b32ad5056603aabbef622f9691f19d0869574dfc610034f18662/gplearn-0.4.1-py3-none-any.whl (41kB)\n",
      "\r",
      "\u001b[K     |████████                        | 10kB 16.3MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████▉                | 20kB 1.9MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████▊        | 30kB 2.5MB/s eta 0:00:01\r",
      "\u001b[K     |███████████████████████████████▊| 40kB 2.8MB/s eta 0:00:01\r",
      "\u001b[K     |████████████████████████████████| 51kB 2.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.6/dist-packages (from gplearn) (0.22.1)\n",
      "Requirement already satisfied: joblib>=0.13.0 in /usr/local/lib/python3.6/dist-packages (from gplearn) (0.14.1)\n",
      "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.20.0->gplearn) (1.17.5)\n",
      "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.20.0->gplearn) (1.4.1)\n",
      "Installing collected packages: gplearn\n",
      "Successfully installed gplearn-0.4.1\n"
     ]
    }
   ],
   "source": [
    "! pip install gplearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "lYpYm3ratorr",
    "outputId": "c885556f-aa56-4146-f0ec-875499ff81d1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 93,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 213
    },
    "colab_type": "code",
    "id": "XV_ZaiCKtee4",
    "outputId": "2e82bdf2-4d01-4c5a-dc77-f15933b8274d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    |   Population Average    |             Best Individual              |\n",
      "---- ------------------------- ------------------------------------------ ----------\n",
      " Gen   Length          Fitness   Length          Fitness      OOB Fitness  Time Left\n",
      "   0    10.14             0.91       22                1                0     43.36m\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>gen_0</th>\n",
       "      <th>gen_1</th>\n",
       "      <th>gen_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>-1.8292</td>\n",
       "      <td>-2.6469</td>\n",
       "      <td>0.5059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>-3.5190</td>\n",
       "      <td>99.1619</td>\n",
       "      <td>3.6243</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>-1.4668</td>\n",
       "      <td>1.3677</td>\n",
       "      <td>3.1826</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target   gen_0    gen_1   gen_2\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227 -1.8292  -2.6469  0.5059\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994 -3.5190  99.1619  3.6243\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910 -1.4668   1.3677  3.1826"
      ]
     },
     "execution_count": 94,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from gplearn.genetic import SymbolicTransformer\n",
    "function_set = ['add', 'sub', 'mul', 'div',\n",
    "                'sqrt', 'log', 'abs', 'neg', 'inv','tan']\n",
    "\n",
    "gp = SymbolicTransformer(generations=800, population_size=200,\n",
    "                         hall_of_fame=100, n_components=10,\n",
    "                         function_set=function_set,\n",
    "                         parsimony_coefficient=0.0005,\n",
    "                         max_samples=0.9, verbose=1,\n",
    "                         random_state=0, n_jobs=6)\n",
    "\n",
    "gen_feats = gp.fit_transform(df.drop(\"target\", axis=1), df[\"target\"]); df.iloc[:,:8]\n",
    "df_out = pd.concat((df,pd.DataFrame(gen_feats, columns=[\"gen_\"+str(a) for a in range(gen_feats.shape[1])])),axis=1); df_out.iloc[:,:8]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "j8E7Oga6wAgR"
   },
   "source": [
    "<a name=\"principal-component\"></a>\n",
    "\n",
    "**>>> Prinicipal Component Features (func)**\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "cxoTQFnWwGti",
    "outputId": "511a5bbc-55a0-4e4f-b153-602432d11666"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 95,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df =df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "KtQghwvAwJT3",
    "outputId": "9d8d2018-d193-4acc-b0da-a5b28b419779"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>a</th>\n",
       "      <th>PCA_1</th>\n",
       "      <th>PCA_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.1227</td>\n",
       "      <td>1.6243</td>\n",
       "      <td>-1.2944</td>\n",
       "      <td>-0.7684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-5.9994</td>\n",
       "      <td>0.8654</td>\n",
       "      <td>1.5375</td>\n",
       "      <td>-0.4537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.5910</td>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2431</td>\n",
       "      <td>1.2220</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   target       a   PCA_1   PCA_2\n",
       "0  1.1227  1.6243 -1.2944 -0.7684\n",
       "1 -5.9994  0.8654  1.5375 -0.4537\n",
       "2 -0.5910  0.3190 -0.2431  1.2220"
      ]
     },
     "execution_count": 96,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.decomposition import PCA, IncrementalPCA\n",
    "\n",
    "def pca_feature(df, memory_issues=False,mem_iss_component=False,variance_or_components=0.80,drop_cols=None):\n",
    "\n",
    "  if memory_issues:\n",
    "    if not mem_iss_component:\n",
    "      raise ValueError(\"If you have memory issues, you have to preselect mem_iss_component\")\n",
    "    pca = IncrementalPCA(mem_iss_component)\n",
    "  else:\n",
    "    if variance_or_components>1:\n",
    "      pca = PCA(n_components=variance_or_components) \n",
    "    else: # automted selection based on variance\n",
    "      pca = PCA(n_components=variance_or_components,svd_solver=\"full\") \n",
    "  X_pca = pca.fit_transform(df.drop(drop_cols,axis=1))\n",
    "  df = pd.concat((df[drop_cols],pd.DataFrame(X_pca, columns=[\"PCA_\"+str(i+1) for i in range(X_pca.shape[1])])),axis=1)\n",
    "  return df\n",
    "\n",
    "df_out = pv.pca_feature(df,variance_or_components=0.80,drop_cols=[\"target\",\"a\"]); df_out\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GvCClmli1Jwj"
   },
   "source": [
    "<a name=\"multiple-lags\"></a>\n",
    "\n",
    "**>>> Multiple Lags (func)**\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "8ttKf5w1G440",
    "outputId": "3563cd69-2970-4853-b607-b513a31ec6dd"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 97,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "O_Ca28oUHESN",
    "outputId": "9194fec4-581f-41d6-d501-10496eb1fca1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>a_t_1</th>\n",
       "      <th>target_t_1</th>\n",
       "      <th>a_t_2</th>\n",
       "      <th>target_t_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>1.6243</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>0.8654</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>1.6243</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target   a_t_1  target_t_1   a_t_2  target_t_2\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227     NaN         NaN     NaN         NaN\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994  1.6243      1.1227     NaN         NaN\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910  0.8654     -5.9994  1.6243      1.1227"
      ]
     },
     "execution_count": 98,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def multiple_lags(df, start=1, end=3,columns=None):\n",
    "  if not columns:\n",
    "    columns = df.columns.to_list()\n",
    "  lags = range(start, end+1)  # Just two lags for demonstration.\n",
    "\n",
    "  df = df.assign(**{\n",
    "      '{}_t_{}'.format(col, t): df[col].shift(t)\n",
    "      for t in lags\n",
    "      for col in columns\n",
    "  })\n",
    "  return df\n",
    "\n",
    "df_out = pv.multiple_lags(df, start=1, end=2,columns=[\"a\",\"target\"]); df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zRZ3Wt0WJcGD"
   },
   "source": [
    "<a name=\"multiple-rolling\"></a>\n",
    "\n",
    "**>>> Multiple Rolling (func)**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "2IQUWAQlJjPs",
    "outputId": "f06af58f-8ebd-4a9b-e324-b6009031fc9d"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910"
      ]
     },
     "execution_count": 99,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy(); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "je5YKWUcJjEa",
    "outputId": "319a8e82-9917-4166-f959-1574e064f6f0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>a_2_mean</th>\n",
       "      <th>a_2_std</th>\n",
       "      <th>a_3_mean</th>\n",
       "      <th>a_3_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>1.2449</td>\n",
       "      <td>0.5367</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>0.5922</td>\n",
       "      <td>0.3863</td>\n",
       "      <td>0.9363</td>\n",
       "      <td>0.6555</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target  a_2_mean  a_2_std  a_3_mean  a_3_std\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227       NaN      NaN       NaN      NaN\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994    1.2449   0.5367       NaN      NaN\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910    0.5922   0.3863    0.9363   0.6555"
      ]
     },
     "execution_count": 100,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def multiple_rolling(df, windows = [1,2], functions=[\"mean\",\"std\"], columns=None):\n",
    "  windows = [1+a for a in windows]\n",
    "  if not columns:\n",
    "    columns = df.columns.to_list()\n",
    "  rolling_dfs = (df[columns].rolling(i)                                    # 1. Create window\n",
    "                  .agg(functions)                                # 1. Aggregate\n",
    "                  .rename({col: '{0}_{1:d}'.format(col, i)\n",
    "                                for col in columns}, axis=1)  # 2. Rename columns\n",
    "                for i in windows)                                # For each window\n",
    "  df_out = pd.concat((df, *rolling_dfs), axis=1)\n",
    "  da = df_out.iloc[:,len(df.columns):]\n",
    "  da = [col[0] + \"_\" + col[1] for col in  da.columns.to_list()]\n",
    "  df_out.columns = df.columns.to_list() + da \n",
    "\n",
    "  return  df_out                      # 3. Concatenate dataframes\n",
    "\n",
    "df_out = pv.multiple_rolling(df, columns=[\"a\"]); df_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6jf68I6hzD9u"
   },
   "source": [
    "<a name=\"data-features\"></a>\n",
    "\n",
    "**>>> Date Features**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "v7oLMcm1yFUM",
    "outputId": "c38fe992-3f9c-4989-c5dc-5e70de09e90d"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>date_fake</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>2019-01-03 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>2019-01-04 12:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>2019-01-06 00:00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target           date_fake\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227 2019-01-03 00:00:00\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994 2019-01-04 12:00:00\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910 2019-01-06 00:00:00"
      ]
     },
     "execution_count": 101,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df[\"date_fake\"] = pd.date_range(start=\"2019-01-03\", end=\"2019-01-06\", periods=len(df)); df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "BR-YMG5Fuf-t",
    "outputId": "215a93a2-b137-4d02-8e71-b0c5cb861393"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>date_fake</th>\n",
       "      <th>date_fake_month</th>\n",
       "      <th>date_fake_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>2019-01-03 00:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>2019-01-04 12:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>2019-01-06 00:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target           date_fake  date_fake_month  date_fake_year\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227 2019-01-03 00:00:00                1            2019\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994 2019-01-04 12:00:00                1            2019\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910 2019-01-06 00:00:00                1            2019"
      ]
     },
     "execution_count": 102,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def date_features(df, date=\"date\"):\n",
    "  df[date] = pd.to_datetime(df[date])\n",
    "  df[date+\"_month\"] = df[date].dt.month.astype(int)\n",
    "  df[date+\"_year\"]  = df[date].dt.year.astype(int)\n",
    "  df[date+\"_week\"]  = df[date].dt.week.astype(int)\n",
    "  df[date+\"_day\"]   = df[date].dt.day.astype(int)\n",
    "  df[date+\"_dayofweek\"]= df[date].dt.dayofweek.astype(int)\n",
    "  df[date+\"_dayofyear\"]= df[date].dt.dayofyear.astype(int)\n",
    "  df[date+\"_hour\"] = df[date].dt.hour.astype(int)\n",
    "  df[date+\"_int\"] = pd.to_datetime(df[date]).astype(int)\n",
    "  return df\n",
    "\n",
    "df_out = date_features(df, date=\"date_fake\"); df_out.iloc[:,:8]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "_Zri3CEDp87s"
   },
   "source": [
    "<a name=\"havervsine-distance\"></a>\n",
    "\n",
    "**>>> Haversine Distance (Location Feature) (func)**\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Z4wkKpbYqZkW"
   },
   "outputs": [],
   "source": [
    "df = df_test.copy()\n",
    "df[\"latitude\"] = [39, 35 , 20]\n",
    "df[\"longitude\"]=  [-77, -40 , -10 ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "iQBLRWpuqFin",
    "outputId": "51114525-4b12-4265-c616-7f1850022beb"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>distance_central</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.1227</td>\n",
       "      <td>39</td>\n",
       "      <td>-77</td>\n",
       "      <td>6702.7127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-5.9994</td>\n",
       "      <td>35</td>\n",
       "      <td>-40</td>\n",
       "      <td>4583.5988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.5910</td>\n",
       "      <td>20</td>\n",
       "      <td>-10</td>\n",
       "      <td>4141.6783</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   target  latitude  longitude  distance_central\n",
       "0  1.1227        39        -77         6702.7127\n",
       "1 -5.9994        35        -40         4583.5988\n",
       "2 -0.5910        20        -10         4141.6783"
      ]
     },
     "execution_count": 104,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from math import sin, cos, sqrt, atan2, radians\n",
    "def haversine_distance(row, lon=\"latitude\", lat=\"longitude\"):\n",
    "    c_lat,c_long = radians(52.5200), radians(13.4050)\n",
    "    R = 6373.0\n",
    "    long = radians(row['longitude'])\n",
    "    lat = radians(row['latitude'])\n",
    "    \n",
    "    dlon = long - c_long\n",
    "    dlat = lat - c_lat\n",
    "    a = sin(dlat / 2)**2 + cos(lat) * cos(c_lat) * sin(dlon / 2)**2\n",
    "    c = 2 * atan2(sqrt(a), sqrt(1 - a))\n",
    "    \n",
    "    return R * c\n",
    "\n",
    "df['distance_central'] = df.apply(pv.haversine_distance,axis=1); df.iloc[:,4:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YM0FzQfbyOlM"
   },
   "source": [
    "<a name=\"parse-address\"></a>\n",
    "\n",
    "**>>> Parse Address**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "gzY78qacyRdM",
    "outputId": "679690b3-e710-4e3e-e0a2-9c5ea0fbd581"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>a</th>\n",
       "      <th>b</th>\n",
       "      <th>c</th>\n",
       "      <th>d</th>\n",
       "      <th>target</th>\n",
       "      <th>addr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.6243</td>\n",
       "      <td>-0.6118</td>\n",
       "      <td>-0.5282</td>\n",
       "      <td>-1.0730</td>\n",
       "      <td>1.1227</td>\n",
       "      <td>Washington, D.C....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.8654</td>\n",
       "      <td>-2.3015</td>\n",
       "      <td>1.7448</td>\n",
       "      <td>-0.7612</td>\n",
       "      <td>-5.9994</td>\n",
       "      <td>Brooklyn, NY 112...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.3190</td>\n",
       "      <td>-0.2494</td>\n",
       "      <td>1.4621</td>\n",
       "      <td>-2.0601</td>\n",
       "      <td>-0.5910</td>\n",
       "      <td>Omaha, NE 68154</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        a       b       c       d  target                 addr\n",
       "0  1.6243 -0.6118 -0.5282 -1.0730  1.1227  Washington, D.C....\n",
       "1  0.8654 -2.3015  1.7448 -0.7612 -5.9994  Brooklyn, NY 112...\n",
       "2  0.3190 -0.2494  1.4621 -2.0601 -0.5910      Omaha, NE 68154"
      ]
     },
     "execution_count": 105,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df_test.copy()\n",
    "df[\"addr\"] = pd.Series([\n",
    "            'Washington, D.C. 20003',\n",
    "            'Brooklyn, NY 11211-1755',\n",
    "            'Omaha, NE 68154' ]) ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "Fp2r22vVyhEL",
    "outputId": "0b809606-8694-4f22-c156-478c8526e0fa"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>state</th>\n",
       "      <th>zip</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Washington</td>\n",
       "      <td>DC</td>\n",
       "      <td>20003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Brooklyn</td>\n",
       "      <td>NY</td>\n",
       "      <td>11211-1755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Omaha</td>\n",
       "      <td>NE</td>\n",
       "      <td>68154</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         city state         zip\n",
       "0  Washington    DC       20003\n",
       "1    Brooklyn    NY  11211-1755\n",
       "2       Omaha    NE       68154"
      ]
     },
     "execution_count": 106,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "regex = (r'(?P<city>[A-Za-z ]+), (?P<state>[A-Z]{2}) (?P<zip>\\d{5}(?:-\\d{4})?)')  \n",
    "\n",
    "df.addr.str.replace('.', '').str.extract(regex)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Pwd3jaFQztxy"
   },
   "source": [
    "<a name=\"processing-strings-in-pandas\"></a>\n",
    "\n",
    "**>>> Processing Strings in Pandas**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "id": "nQ-pdvmW8dg1",
    "outputId": "552d593a-2e51-4312-c116-08c871240470"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>foo1 foo1</td>\n",
       "      <td>2009-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>foo2 foo2</td>\n",
       "      <td>2009-01-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>foo3 foo3</td>\n",
       "      <td>2009-01-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>foo4 foo4</td>\n",
       "      <td>2009-01-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>foo5 foo5</td>\n",
       "      <td>2009-01-07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     A    B          C          D\n",
       "0  0.0  0.0  foo1 foo1 2009-01-01\n",
       "1  1.0  1.0  foo2 foo2 2009-01-02\n",
       "2  2.0  0.0  foo3 foo3 2009-01-05\n",
       "3  3.0  1.0  foo4 foo4 2009-01-06\n",
       "4  4.0  0.0  foo5 foo5 2009-01-07"
      ]
     },
     "execution_count": 107,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.util.testing.makeMixedDataFrame()\n",
    "df[\"C\"] = df[\"C\"] + \" \" + df[\"C\"] ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "colab_type": "code",
    "id": "Vc7n5t1UzwwP",
    "outputId": "87f15d38-14ab-4401-ed0f-e61c15f39da6"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>this is a regula...</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https:/</td>\n",
       "      <td>docs.p.org</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     0           1\n",
       "0  this is a regula...        None\n",
       "1              https:/  docs.p.org\n",
       "2                  NaN         NaN"
      ]
     },
     "execution_count": 108,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"convert column to UPPERCASE\"\"\"\n",
    "\n",
    "col_name = \"C\"\n",
    "df[col_name].str.upper()\n",
    "\n",
    "\"\"\"count string occurence in each row\"\"\"\n",
    "df[col_name].str.count(r'\\d') # counts number of digits\n",
    "\n",
    "\"\"\"count # o chars in each row\"\"\"\n",
    "df[col_name].str.count('o') # counts number of digits\n",
    "\n",
    "\"\"\"split rows\"\"\"\n",
    "s = pd.Series([\"this is a regular sentence\", \"https://docs.p.org\", np.nan])\n",
    "s.str.split()\n",
    "\n",
    "\"\"\"this creates new columns with the different split values (instead of lists)\"\"\"\n",
    "s.str.split(expand=True)  \n",
    "\n",
    "\"\"\"limit the number of splits to 1, and start spliting from the rights side\"\"\"\n",
    "s.str.rsplit(\"/\", n=1, expand=True) \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Jjhp059utrsb"
   },
   "source": [
    "<a name=\"filtering-strings-in-pandas\"></a>\n",
    "\n",
    "**>>> Filtering Strings in Pandas**\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 206
    },
    "colab_type": "code",
    "id": "R9ITTnm8-DHd",
    "outputId": "7a479c55-437f-4d1c-dcea-81fcad1cd889"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>foo1 foo1</td>\n",
       "      <td>2009-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>foo2 foo2</td>\n",
       "      <td>2009-01-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>foo3 foo3</td>\n",
       "      <td>2009-01-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>foo4 foo4</td>\n",
       "      <td>2009-01-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>foo5 foo5</td>\n",
       "      <td>2009-01-07</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     A    B          C          D\n",
       "0  0.0  0.0  foo1 foo1 2009-01-01\n",
       "1  1.0  1.0  foo2 foo2 2009-01-02\n",
       "2  2.0  0.0  foo3 foo3 2009-01-05\n",
       "3  3.0  1.0  foo4 foo4 2009-01-06\n",
       "4  4.0  0.0  foo5 foo5 2009-01-07"
      ]
     },
     "execution_count": 109,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.util.testing.makeMixedDataFrame()\n",
    "df[\"C\"] = df[\"C\"] + \" \" + df[\"C\"] ; df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KlTMu6tuz29K"
   },
   "outputs": [],
   "source": [
    "col_name = \"C\"\n",
    "\n",
    "\"\"\"check if a certain word/pattern occurs in each row\"\"\"\n",
    "df[col_name].str.contains('oo')  # returns True/False for each row\n",
    "\n",
    "\"\"\"find occurences\"\"\"\n",
    "df[col_name].str.findall(r'[ABC]\\d') # returns a list of the found occurences of the specified pattern for each row\n",
    "\n",
    "\"\"\"replace Weekdays by abbrevations (e.g. Monday --> Mon)\"\"\"\n",
    "df[col_name].str.replace(r'(\\w+day\\b)', lambda x: x.groups[0][:3]) # () in r'' creates a group with one element, which we acces with x.groups[0]\n",
    "\n",
    "\"\"\"create dataframe from regex groups (str.extract() uses first match of the pattern only)\"\"\"\n",
    "df[col_name].str.extract(r'(\\d?\\d):(\\d\\d)')\n",
    "df[col_name].str.extract(r'(?P<hours>\\d?\\d):(?P<minutes>\\d\\d)')\n",
    "df[col_name].str.extract(r'(?P<time>(?P<hours>\\d?\\d):(?P<minutes>\\d\\d))')\n",
    "\n",
    "\"\"\"if you want to take into account ALL matches in a row (not only first one):\"\"\"\n",
    "df[col_name].str.extractall(r'(\\d?\\d):(\\d\\d)') # this generates a multiindex with level 1 = 'match', indicating the order of the match\n",
    "\n",
    "df[col_name].replace('\\n', '', regex=True, inplace=True)\n",
    "\n",
    "\"\"\"remove all the characters after &# (including &#) for column - col_1\"\"\"\n",
    "df[col_name].replace(' &#.*', '', regex=True, inplace=True)\n",
    "\n",
    "\"\"\"remove white space at the beginning of string\"\"\"\n",
    "df[col_name] = df[col_name].str.lstrip()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BbfDs5hnHCOe"
   },
   "source": [
    "<a name=\"model-validation\"></a>\n",
    "\n",
    "## **Model Validation**\n",
    "\n",
    "\n",
    "---\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QyVhiNsMPHf6"
   },
   "source": [
    "\n",
    "<a name=\"classification-metrics\"></a>\n",
    "**>>> Classification Metrics (func)**\n",
    "\n",
    "\n",
    "\n",
    "---\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lUR5EMG6NJyu"
   },
   "outputs": [],
   "source": [
    "y_test = [0, 1, 1, 1, 0]\n",
    "y_predict = [0, 0, 1, 1, 1]\n",
    "y_prob = [0.2,0.6,0.7,0.7,0.9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 680
    },
    "colab_type": "code",
    "id": "q9adwyJ9M0sa",
    "outputId": "d7e28373-3fed-4fa3-82c8-23cff25f8046"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6\n",
      "0.5\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Metrics</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Average Log Likelihood</th>\n",
       "      <td>0.7500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Brier Score Loss</th>\n",
       "      <td>0.2380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Accuracy Score</th>\n",
       "      <td>0.6000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ROC AUC Sore</th>\n",
       "      <td>0.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Average Precision Score</th>\n",
       "      <td>0.6944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precision - Bankrupt Firms</th>\n",
       "      <td>0.6667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>False Positive Rate (p-value)</th>\n",
       "      <td>0.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Precision - Healthy Firms</th>\n",
       "      <td>0.5000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>False Negative Rate (recall error)</th>\n",
       "      <td>0.3333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>False Discovery Rate</th>\n",
       "      <td>0.3333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>All Observations</th>\n",
       "      <td>5.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bankruptcy Sample</th>\n",
       "      <td>3.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Healthy Sample</th>\n",
       "      <td>2.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recalled Bankruptcy</th>\n",
       "      <td>3.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Correct (True Positives)</th>\n",
       "      <td>2.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Incorrect (False Positives)</th>\n",
       "      <td>1.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recalled Healthy</th>\n",
       "      <td>2.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Correct (True Negatives)</th>\n",
       "      <td>1.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Incorrect (False Negatives)</th>\n",
       "      <td>1.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Metrics\n",
       "Average Log Likel...   0.7500\n",
       "Brier Score Loss       0.2380\n",
       "Accuracy Score         0.6000\n",
       "ROC AUC Sore           0.5000\n",
       "Average Precision...   0.6944\n",
       "Precision - Bankr...   0.6667\n",
       "False Positive Ra...   0.5000\n",
       "Precision - Healt...   0.5000\n",
       "False Negative Ra...   0.3333\n",
       "False Discovery R...   0.3333\n",
       "All Observations       5.0000\n",
       "Bankruptcy Sample      3.0000\n",
       "Healthy Sample         2.0000\n",
       "Recalled Bankruptcy    3.0000\n",
       "Correct (True Pos...   2.0000\n",
       "Incorrect (False ...   1.0000\n",
       "Recalled Healthy       2.0000\n",
       "Correct (True Neg...   1.0000\n",
       "Incorrect (False ...   1.0000"
      ]
     },
     "execution_count": 112,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score, average_precision_score, confusion_matrix\n",
    "from sklearn.metrics import log_loss, brier_score_loss, accuracy_score\n",
    "\n",
    "def classification_scores(y_test, y_predict, y_prob):\n",
    "\n",
    "  confusion_mat = confusion_matrix(y_test,y_predict)\n",
    "\n",
    "  TN = confusion_mat[0][0]\n",
    "  FP = confusion_mat[0][1]\n",
    "  TP = confusion_mat[1][1]\n",
    "  FN = confusion_mat[1][0]\n",
    "\n",
    "  TPR = TP/(TP+FN)\n",
    "  # Specificity or true negative rate\n",
    "  TNR = TN/(TN+FP) \n",
    "  # Precision or positive predictive value\n",
    "  PPV = TP/(TP+FP)\n",
    "  # Negative predictive value\n",
    "  NPV = TN/(TN+FN)\n",
    "  # Fall out or false positive rate\n",
    "  FPR = FP/(FP+TN)\n",
    "  # False negative rate\n",
    "  FNR = FN/(TP+FN)\n",
    "  # False discovery rate\n",
    "  FDR = FP/(TP+FP)\n",
    "\n",
    "  ll = log_loss(y_test, y_prob) # Its low but means nothing to me. \n",
    "  br = brier_score_loss(y_test, y_prob) # Its low but means nothing to me. \n",
    "  acc = accuracy_score(y_test, y_predict)\n",
    "  print(acc)\n",
    "  auc = roc_auc_score(y_test, y_prob)\n",
    "  print(auc)\n",
    "  prc = average_precision_score(y_test, y_prob) \n",
    "\n",
    "  data = np.array([np.arange(1)]*1).T\n",
    "\n",
    "  df_exec = pd.DataFrame(data)\n",
    "\n",
    "  df_exec[\"Average Log Likelihood\"] = ll\n",
    "  df_exec[\"Brier Score Loss\"] = br\n",
    "  df_exec[\"Accuracy Score\"] = acc\n",
    "  df_exec[\"ROC AUC Sore\"] = auc\n",
    "  df_exec[\"Average Precision Score\"] = prc\n",
    "  df_exec[\"Precision - Bankrupt Firms\"] = PPV\n",
    "  df_exec[\"False Positive Rate (p-value)\"] = FPR\n",
    "  df_exec[\"Precision - Healthy Firms\"] = NPV\n",
    "  df_exec[\"False Negative Rate (recall error)\"] = FNR\n",
    "  df_exec[\"False Discovery Rate \"] = FDR\n",
    "  df_exec[\"All Observations\"] = TN + TP + FN + FP\n",
    "  df_exec[\"Bankruptcy Sample\"] = TP + FN\n",
    "  df_exec[\"Healthy Sample\"] = TN + FP\n",
    "  df_exec[\"Recalled Bankruptcy\"] = TP + FP\n",
    "  df_exec[\"Correct (True Positives)\"] = TP\n",
    "  df_exec[\"Incorrect (False Positives)\"] = FP\n",
    "  df_exec[\"Recalled Healthy\"] = TN + FN\n",
    "  df_exec[\"Correct (True Negatives)\"] = TN\n",
    "  df_exec[\"Incorrect (False Negatives)\"] = FN\n",
    "\n",
    "  df_exec = df_exec.T[1:]\n",
    "  df_exec.columns = [\"Metrics\"]\n",
    "  return df_exec\n",
    "\n",
    "\n",
    "met = pv.classification_scores(y_test, y_predict, y_prob); met"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Pandas Vault.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}