basic_ml_on_pyspark.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "dbc63241-ffbf-4656-9f4b-8fcb5098eb31",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>emp_no</th><th>emp_level</th><th>birth_date</th><th>first_name</th><th>last_name</th><th>sex</th><th>hire_date</th><th>salary</th><th>department</th></tr></thead><tbody><tr><td>10001</td><td>Staff</td><td>1953-07-25</td><td>Hideyuki</td><td>Zallocco</td><td>M</td><td>1990-04-28</td><td>60117</td><td>Development</td></tr><tr><td>10002</td><td>Engineer</td><td>1954-11-18</td><td>Byong</td><td>Delgrande</td><td>F</td><td>1991-09-07</td><td>65828</td><td>Sales</td></tr><tr><td>10003</td><td>Engineer</td><td>1958-01-30</td><td>Berry</td><td>Babb</td><td>F</td><td>1992-03-21</td><td>40006</td><td>Production</td></tr><tr><td>10004</td><td>Staff</td><td>1957-09-28</td><td>Xiong</td><td>Verhoeff</td><td>M</td><td>1987-11-26</td><td>40054</td><td>Production</td></tr><tr><td>10005</td><td>Senior Engineer</td><td>1952-10-28</td><td>Abdelkader</td><td>Baumann</td><td>F</td><td>1991-01-18</td><td>78228</td><td>Human Resources</td></tr><tr><td>10006</td><td>Senior Engineer</td><td>1959-10-30</td><td>Eran</td><td>Cusworth</td><td>M</td><td>1986-11-14</td><td>40000</td><td>Development</td></tr><tr><td>10007</td><td>Staff</td><td>1957-04-14</td><td>Christoph</td><td>Parfitt</td><td>M</td><td>1991-06-28</td><td>56724</td><td>Research</td></tr><tr><td>10008</td><td>Senior Staff</td><td>1964-11-17</td><td>Xudong</td><td>Samarati</td><td>M</td><td>1985-11-13</td><td>46671</td><td>Development</td></tr><tr><td>10009</td><td>Engineer</td><td>1962-12-18</td><td>Lihong</td><td>Magliocco</td><td>M</td><td>1993-10-23</td><td>60929</td><td>Quality Management</td></tr><tr><td>10010</td><td>Senior Staff</td><td>1956-04-24</td><td>Kwangyoen</td><td>Speek</td><td>F</td><td>1993-02-14</td><td>72488</td><td>Production</td></tr><tr><td>10010</td><td>Senior Staff</td><td>1956-04-24</td><td>Kwangyoen</td><td>Speek</td><td>F</td><td>1993-02-14</td><td>72488</td><td>Quality Management</td></tr><tr><td>10011</td><td>Engineer</td><td>1961-12-31</td><td>Shuichi</td><td>Tyugu</td><td>F</td><td>1995-01-17</td><td>42365</td><td>Customer Service</td></tr><tr><td>10012</td><td>Senior Engineer</td><td>1961-10-25</td><td>Abdulah</td><td>Lunn</td><td>M</td><td>1989-04-08</td><td>40000</td><td>Development</td></tr><tr><td>10013</td><td>Senior Engineer</td><td>1961-08-30</td><td>Perry</td><td>Lorho</td><td>F</td><td>1991-08-03</td><td>40000</td><td>Human Resources</td></tr><tr><td>10014</td><td>Staff</td><td>1958-03-23</td><td>Subbu</td><td>Bultermann</td><td>F</td><td>1988-03-25</td><td>46168</td><td>Development</td></tr><tr><td>10015</td><td>Engineer</td><td>1959-05-19</td><td>Bojan</td><td>Zallocco</td><td>M</td><td>1986-10-14</td><td>40000</td><td>Research</td></tr><tr><td>10016</td><td>Technique Leader</td><td>1963-07-12</td><td>Bilhanan</td><td>Wuwongse</td><td>M</td><td>1993-10-06</td><td>70889</td><td>Sales</td></tr><tr><td>10017</td><td>Staff</td><td>1964-01-19</td><td>Venkatesan</td><td>Gilg</td><td>M</td><td>1993-06-28</td><td>71380</td><td>Marketing</td></tr><tr><td>10018</td><td>Engineer</td><td>1957-12-11</td><td>Naftali</td><td>Dulli</td><td>M</td><td>1993-06-06</td><td>55881</td><td>Production</td></tr><tr><td>10018</td><td>Engineer</td><td>1957-12-11</td><td>Naftali</td><td>Dulli</td><td>M</td><td>1993-06-06</td><td>55881</td><td>Development</td></tr></tbody></table></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "aggData": [],
       "aggError": "",
       "aggOverflow": false,
       "aggSchema": [],
       "aggSeriesLimitReached": false,
       "aggType": "",
       "arguments": {},
       "columnCustomDisplayInfos": {},
       "data": [
        [
         10001,
         "Staff",
         "1953-07-25",
         "Hideyuki",
         "Zallocco",
         "M",
         "1990-04-28",
         60117,
         "Development"
        ],
        [
         10002,
         "Engineer",
         "1954-11-18",
         "Byong",
         "Delgrande",
         "F",
         "1991-09-07",
         65828,
         "Sales"
        ],
        [
         10003,
         "Engineer",
         "1958-01-30",
         "Berry",
         "Babb",
         "F",
         "1992-03-21",
         40006,
         "Production"
        ],
        [
         10004,
         "Staff",
         "1957-09-28",
         "Xiong",
         "Verhoeff",
         "M",
         "1987-11-26",
         40054,
         "Production"
        ],
        [
         10005,
         "Senior Engineer",
         "1952-10-28",
         "Abdelkader",
         "Baumann",
         "F",
         "1991-01-18",
         78228,
         "Human Resources"
        ],
        [
         10006,
         "Senior Engineer",
         "1959-10-30",
         "Eran",
         "Cusworth",
         "M",
         "1986-11-14",
         40000,
         "Development"
        ],
        [
         10007,
         "Staff",
         "1957-04-14",
         "Christoph",
         "Parfitt",
         "M",
         "1991-06-28",
         56724,
         "Research"
        ],
        [
         10008,
         "Senior Staff",
         "1964-11-17",
         "Xudong",
         "Samarati",
         "M",
         "1985-11-13",
         46671,
         "Development"
        ],
        [
         10009,
         "Engineer",
         "1962-12-18",
         "Lihong",
         "Magliocco",
         "M",
         "1993-10-23",
         60929,
         "Quality Management"
        ],
        [
         10010,
         "Senior Staff",
         "1956-04-24",
         "Kwangyoen",
         "Speek",
         "F",
         "1993-02-14",
         72488,
         "Production"
        ],
        [
         10010,
         "Senior Staff",
         "1956-04-24",
         "Kwangyoen",
         "Speek",
         "F",
         "1993-02-14",
         72488,
         "Quality Management"
        ],
        [
         10011,
         "Engineer",
         "1961-12-31",
         "Shuichi",
         "Tyugu",
         "F",
         "1995-01-17",
         42365,
         "Customer Service"
        ],
        [
         10012,
         "Senior Engineer",
         "1961-10-25",
         "Abdulah",
         "Lunn",
         "M",
         "1989-04-08",
         40000,
         "Development"
        ],
        [
         10013,
         "Senior Engineer",
         "1961-08-30",
         "Perry",
         "Lorho",
         "F",
         "1991-08-03",
         40000,
         "Human Resources"
        ],
        [
         10014,
         "Staff",
         "1958-03-23",
         "Subbu",
         "Bultermann",
         "F",
         "1988-03-25",
         46168,
         "Development"
        ],
        [
         10015,
         "Engineer",
         "1959-05-19",
         "Bojan",
         "Zallocco",
         "M",
         "1986-10-14",
         40000,
         "Research"
        ],
        [
         10016,
         "Technique Leader",
         "1963-07-12",
         "Bilhanan",
         "Wuwongse",
         "M",
         "1993-10-06",
         70889,
         "Sales"
        ],
        [
         10017,
         "Staff",
         "1964-01-19",
         "Venkatesan",
         "Gilg",
         "M",
         "1993-06-28",
         71380,
         "Marketing"
        ],
        [
         10018,
         "Engineer",
         "1957-12-11",
         "Naftali",
         "Dulli",
         "M",
         "1993-06-06",
         55881,
         "Production"
        ],
        [
         10018,
         "Engineer",
         "1957-12-11",
         "Naftali",
         "Dulli",
         "M",
         "1993-06-06",
         55881,
         "Development"
        ]
       ],
       "datasetInfos": [],
       "dbfsResultPath": null,
       "isJsonSchema": true,
       "metadata": {},
       "overflow": false,
       "plotOptions": {
        "customPlotOptions": {},
        "displayType": "table",
        "pivotAggregation": null,
        "pivotColumns": null,
        "xColumns": null,
        "yColumns": null
       },
       "removedWidgets": [],
       "schema": [
        {
         "metadata": "{}",
         "name": "emp_no",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "emp_level",
         "type": "\"string\""
        },
        {
         "metadata": "{\"__detected_date_formats\":\"yyyy-M-d\"}",
         "name": "birth_date",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "first_name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "last_name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "sex",
         "type": "\"string\""
        },
        {
         "metadata": "{\"__detected_date_formats\":\"yyyy-M-d\"}",
         "name": "hire_date",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "salary",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "department",
         "type": "\"string\""
        }
       ],
       "type": "table"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "file_location = \"/FileStore/tables/data/full_employee.csv\"\n",
    "file_type = \"csv\"\n",
    "\n",
    "emp_df = spark.read.csv(file_location, header=True, inferSchema=True)\n",
    "\n",
    "display(emp_df.limit(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "bb164518-ce5a-470e-8dd0-07eedaf9c3a7",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Out[90]: [331603, 9]"
     ]
    }
   ],
   "source": [
    "def shape(df):\n",
    "    row_num = df.count()\n",
    "    col_num = len(df.columns)\n",
    "    return [row_num, col_num]\n",
    "\n",
    "shape(emp_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "14b2c8fc-b3f2-4eb9-8cff-751f19862360",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n |-- emp_no: integer (nullable = true)\n |-- emp_level: string (nullable = true)\n |-- birth_date: date (nullable = true)\n |-- first_name: string (nullable = true)\n |-- last_name: string (nullable = true)\n |-- sex: string (nullable = true)\n |-- hire_date: date (nullable = true)\n |-- salary: integer (nullable = true)\n |-- department: string (nullable = true)\n\n"
     ]
    }
   ],
   "source": [
    "emp_df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "981e1bff-1fe5-48f6-b852-d3b5ca663d7e",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------+---------+----------+----------+---------+---+---------+------+----------+\n|emp_no|emp_level|birth_date|first_name|last_name|sex|hire_date|salary|department|\n+------+---------+----------+----------+---------+---+---------+------+----------+\n|     0|        0|         0|         0|        0|  0|        0|     0|         0|\n+------+---------+----------+----------+---------+---+---------+------+----------+\n\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql.functions import *\n",
    "\n",
    "null_counter = emp_df.select([\n",
    "    sum(when(col(coll).isNull(), 1).otherwise(0)).alias(coll) for coll in emp_df.columns\n",
    "])\n",
    "\n",
    "null_counter.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "9fffff2b-9260-46bd-bd51-51baebcd1415",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "source": [
    "BASIC ML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "c07a3107-e96c-4004-8388-b0b4fe99f284",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import VectorAssembler, StringIndexer\n",
    "#transform strings to int\n",
    "\n",
    "ind = StringIndexer(inputCols =['emp_level','sex', \"department\"], outputCols = ['level_indexed', 'sex_indexed', 'dept_indexed'])\n",
    "\n",
    "emp_df = ind.fit(emp_df).transform(emp_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "a677c449-e835-4451-aa14-91aeb32c89ac",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>emp_no</th><th>emp_level</th><th>birth_date</th><th>first_name</th><th>last_name</th><th>sex</th><th>hire_date</th><th>salary</th><th>department</th><th>level_indexed</th><th>sex_indexed</th><th>dept_indexed</th></tr></thead><tbody><tr><td>10001</td><td>Staff</td><td>1953-07-25</td><td>Hideyuki</td><td>Zallocco</td><td>M</td><td>1990-04-28</td><td>60117</td><td>Development</td><td>0.0</td><td>0.0</td><td>0.0</td></tr><tr><td>10002</td><td>Engineer</td><td>1954-11-18</td><td>Byong</td><td>Delgrande</td><td>F</td><td>1991-09-07</td><td>65828</td><td>Sales</td><td>2.0</td><td>1.0</td><td>2.0</td></tr><tr><td>10003</td><td>Engineer</td><td>1958-01-30</td><td>Berry</td><td>Babb</td><td>F</td><td>1992-03-21</td><td>40006</td><td>Production</td><td>2.0</td><td>1.0</td><td>1.0</td></tr><tr><td>10004</td><td>Staff</td><td>1957-09-28</td><td>Xiong</td><td>Verhoeff</td><td>M</td><td>1987-11-26</td><td>40054</td><td>Production</td><td>0.0</td><td>0.0</td><td>1.0</td></tr><tr><td>10005</td><td>Senior Engineer</td><td>1952-10-28</td><td>Abdelkader</td><td>Baumann</td><td>F</td><td>1991-01-18</td><td>78228</td><td>Human Resources</td><td>1.0</td><td>1.0</td><td>7.0</td></tr><tr><td>10006</td><td>Senior Engineer</td><td>1959-10-30</td><td>Eran</td><td>Cusworth</td><td>M</td><td>1986-11-14</td><td>40000</td><td>Development</td><td>1.0</td><td>0.0</td><td>0.0</td></tr><tr><td>10007</td><td>Staff</td><td>1957-04-14</td><td>Christoph</td><td>Parfitt</td><td>M</td><td>1991-06-28</td><td>56724</td><td>Research</td><td>0.0</td><td>0.0</td><td>4.0</td></tr><tr><td>10008</td><td>Senior Staff</td><td>1964-11-17</td><td>Xudong</td><td>Samarati</td><td>M</td><td>1985-11-13</td><td>46671</td><td>Development</td><td>3.0</td><td>0.0</td><td>0.0</td></tr><tr><td>10009</td><td>Engineer</td><td>1962-12-18</td><td>Lihong</td><td>Magliocco</td><td>M</td><td>1993-10-23</td><td>60929</td><td>Quality Management</td><td>2.0</td><td>0.0</td><td>6.0</td></tr><tr><td>10010</td><td>Senior Staff</td><td>1956-04-24</td><td>Kwangyoen</td><td>Speek</td><td>F</td><td>1993-02-14</td><td>72488</td><td>Production</td><td>3.0</td><td>1.0</td><td>1.0</td></tr><tr><td>10010</td><td>Senior Staff</td><td>1956-04-24</td><td>Kwangyoen</td><td>Speek</td><td>F</td><td>1993-02-14</td><td>72488</td><td>Quality Management</td><td>3.0</td><td>1.0</td><td>6.0</td></tr><tr><td>10011</td><td>Engineer</td><td>1961-12-31</td><td>Shuichi</td><td>Tyugu</td><td>F</td><td>1995-01-17</td><td>42365</td><td>Customer Service</td><td>2.0</td><td>1.0</td><td>3.0</td></tr><tr><td>10012</td><td>Senior Engineer</td><td>1961-10-25</td><td>Abdulah</td><td>Lunn</td><td>M</td><td>1989-04-08</td><td>40000</td><td>Development</td><td>1.0</td><td>0.0</td><td>0.0</td></tr><tr><td>10013</td><td>Senior Engineer</td><td>1961-08-30</td><td>Perry</td><td>Lorho</td><td>F</td><td>1991-08-03</td><td>40000</td><td>Human Resources</td><td>1.0</td><td>1.0</td><td>7.0</td></tr><tr><td>10014</td><td>Staff</td><td>1958-03-23</td><td>Subbu</td><td>Bultermann</td><td>F</td><td>1988-03-25</td><td>46168</td><td>Development</td><td>0.0</td><td>1.0</td><td>0.0</td></tr><tr><td>10015</td><td>Engineer</td><td>1959-05-19</td><td>Bojan</td><td>Zallocco</td><td>M</td><td>1986-10-14</td><td>40000</td><td>Research</td><td>2.0</td><td>0.0</td><td>4.0</td></tr><tr><td>10016</td><td>Technique Leader</td><td>1963-07-12</td><td>Bilhanan</td><td>Wuwongse</td><td>M</td><td>1993-10-06</td><td>70889</td><td>Sales</td><td>4.0</td><td>0.0</td><td>2.0</td></tr><tr><td>10017</td><td>Staff</td><td>1964-01-19</td><td>Venkatesan</td><td>Gilg</td><td>M</td><td>1993-06-28</td><td>71380</td><td>Marketing</td><td>0.0</td><td>0.0</td><td>5.0</td></tr><tr><td>10018</td><td>Engineer</td><td>1957-12-11</td><td>Naftali</td><td>Dulli</td><td>M</td><td>1993-06-06</td><td>55881</td><td>Production</td><td>2.0</td><td>0.0</td><td>1.0</td></tr><tr><td>10018</td><td>Engineer</td><td>1957-12-11</td><td>Naftali</td><td>Dulli</td><td>M</td><td>1993-06-06</td><td>55881</td><td>Development</td><td>2.0</td><td>0.0</td><td>0.0</td></tr></tbody></table></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "aggData": [],
       "aggError": "",
       "aggOverflow": false,
       "aggSchema": [],
       "aggSeriesLimitReached": false,
       "aggType": "",
       "arguments": {},
       "columnCustomDisplayInfos": {},
       "data": [
        [
         10001,
         "Staff",
         "1953-07-25",
         "Hideyuki",
         "Zallocco",
         "M",
         "1990-04-28",
         60117,
         "Development",
         0.0,
         0.0,
         0.0
        ],
        [
         10002,
         "Engineer",
         "1954-11-18",
         "Byong",
         "Delgrande",
         "F",
         "1991-09-07",
         65828,
         "Sales",
         2.0,
         1.0,
         2.0
        ],
        [
         10003,
         "Engineer",
         "1958-01-30",
         "Berry",
         "Babb",
         "F",
         "1992-03-21",
         40006,
         "Production",
         2.0,
         1.0,
         1.0
        ],
        [
         10004,
         "Staff",
         "1957-09-28",
         "Xiong",
         "Verhoeff",
         "M",
         "1987-11-26",
         40054,
         "Production",
         0.0,
         0.0,
         1.0
        ],
        [
         10005,
         "Senior Engineer",
         "1952-10-28",
         "Abdelkader",
         "Baumann",
         "F",
         "1991-01-18",
         78228,
         "Human Resources",
         1.0,
         1.0,
         7.0
        ],
        [
         10006,
         "Senior Engineer",
         "1959-10-30",
         "Eran",
         "Cusworth",
         "M",
         "1986-11-14",
         40000,
         "Development",
         1.0,
         0.0,
         0.0
        ],
        [
         10007,
         "Staff",
         "1957-04-14",
         "Christoph",
         "Parfitt",
         "M",
         "1991-06-28",
         56724,
         "Research",
         0.0,
         0.0,
         4.0
        ],
        [
         10008,
         "Senior Staff",
         "1964-11-17",
         "Xudong",
         "Samarati",
         "M",
         "1985-11-13",
         46671,
         "Development",
         3.0,
         0.0,
         0.0
        ],
        [
         10009,
         "Engineer",
         "1962-12-18",
         "Lihong",
         "Magliocco",
         "M",
         "1993-10-23",
         60929,
         "Quality Management",
         2.0,
         0.0,
         6.0
        ],
        [
         10010,
         "Senior Staff",
         "1956-04-24",
         "Kwangyoen",
         "Speek",
         "F",
         "1993-02-14",
         72488,
         "Production",
         3.0,
         1.0,
         1.0
        ],
        [
         10010,
         "Senior Staff",
         "1956-04-24",
         "Kwangyoen",
         "Speek",
         "F",
         "1993-02-14",
         72488,
         "Quality Management",
         3.0,
         1.0,
         6.0
        ],
        [
         10011,
         "Engineer",
         "1961-12-31",
         "Shuichi",
         "Tyugu",
         "F",
         "1995-01-17",
         42365,
         "Customer Service",
         2.0,
         1.0,
         3.0
        ],
        [
         10012,
         "Senior Engineer",
         "1961-10-25",
         "Abdulah",
         "Lunn",
         "M",
         "1989-04-08",
         40000,
         "Development",
         1.0,
         0.0,
         0.0
        ],
        [
         10013,
         "Senior Engineer",
         "1961-08-30",
         "Perry",
         "Lorho",
         "F",
         "1991-08-03",
         40000,
         "Human Resources",
         1.0,
         1.0,
         7.0
        ],
        [
         10014,
         "Staff",
         "1958-03-23",
         "Subbu",
         "Bultermann",
         "F",
         "1988-03-25",
         46168,
         "Development",
         0.0,
         1.0,
         0.0
        ],
        [
         10015,
         "Engineer",
         "1959-05-19",
         "Bojan",
         "Zallocco",
         "M",
         "1986-10-14",
         40000,
         "Research",
         2.0,
         0.0,
         4.0
        ],
        [
         10016,
         "Technique Leader",
         "1963-07-12",
         "Bilhanan",
         "Wuwongse",
         "M",
         "1993-10-06",
         70889,
         "Sales",
         4.0,
         0.0,
         2.0
        ],
        [
         10017,
         "Staff",
         "1964-01-19",
         "Venkatesan",
         "Gilg",
         "M",
         "1993-06-28",
         71380,
         "Marketing",
         0.0,
         0.0,
         5.0
        ],
        [
         10018,
         "Engineer",
         "1957-12-11",
         "Naftali",
         "Dulli",
         "M",
         "1993-06-06",
         55881,
         "Production",
         2.0,
         0.0,
         1.0
        ],
        [
         10018,
         "Engineer",
         "1957-12-11",
         "Naftali",
         "Dulli",
         "M",
         "1993-06-06",
         55881,
         "Development",
         2.0,
         0.0,
         0.0
        ]
       ],
       "datasetInfos": [],
       "dbfsResultPath": null,
       "isJsonSchema": true,
       "metadata": {},
       "overflow": false,
       "plotOptions": {
        "customPlotOptions": {},
        "displayType": "table",
        "pivotAggregation": null,
        "pivotColumns": null,
        "xColumns": null,
        "yColumns": null
       },
       "removedWidgets": [],
       "schema": [
        {
         "metadata": "{}",
         "name": "emp_no",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "emp_level",
         "type": "\"string\""
        },
        {
         "metadata": "{\"__detected_date_formats\":\"yyyy-M-d\"}",
         "name": "birth_date",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "first_name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "last_name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "sex",
         "type": "\"string\""
        },
        {
         "metadata": "{\"__detected_date_formats\":\"yyyy-M-d\"}",
         "name": "hire_date",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "salary",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "department",
         "type": "\"string\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"Staff\",\"Senior Engineer\",\"Engineer\",\"Senior Staff\",\"Technique Leader\",\"Assistant Engineer\",\"Manager\"],\"type\":\"nominal\",\"name\":\"level_indexed\"}}",
         "name": "level_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"M\",\"F\"],\"type\":\"nominal\",\"name\":\"sex_indexed\"}}",
         "name": "sex_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"Development\",\"Production\",\"Sales\",\"Customer Service\",\"Research\",\"Marketing\",\"Quality Management\",\"Human Resources\",\"Finance\"],\"type\":\"nominal\",\"name\":\"dept_indexed\"}}",
         "name": "dept_indexed",
         "type": "\"double\""
        }
       ],
       "type": "table"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(emp_df.limit(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "0cbfdd24-761c-4424-80f5-a0a56a1263d1",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Out[95]: [('emp_no', 'int'),\n ('emp_level', 'string'),\n ('birth_date', 'date'),\n ('first_name', 'string'),\n ('last_name', 'string'),\n ('sex', 'string'),\n ('hire_date', 'date'),\n ('salary', 'int'),\n ('department', 'string'),\n ('level_indexed', 'double'),\n ('sex_indexed', 'double'),\n ('dept_indexed', 'double')]"
     ]
    }
   ],
   "source": [
    "emp_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "b8b76398-81c7-4c26-8aa9-1a03cede35ec",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "emp_df = emp_df.select(\n",
    "    \"*\",\n",
    "    year(col('birth_date')).alias(\"birth_year\"),\n",
    "    year(col('hire_date')).alias(\"hire_year\")    \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "d52084e1-f922-4376-85c1-a6ee6596153b",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>emp_no</th><th>emp_level</th><th>birth_date</th><th>first_name</th><th>last_name</th><th>sex</th><th>hire_date</th><th>salary</th><th>department</th><th>level_indexed</th><th>sex_indexed</th><th>dept_indexed</th><th>birth_year</th><th>hire_year</th><th>year_union</th><th>scaled_year</th></tr></thead><tbody><tr><td>10001</td><td>Staff</td><td>1953-07-25</td><td>Hideyuki</td><td>Zallocco</td><td>M</td><td>1990-04-28</td><td>60117</td><td>Development</td><td>0.0</td><td>0.0</td><td>0.0</td><td>1953</td><td>1990</td><td>Map(vectorType -> dense, length -> 2, values -> List(1953.0, 1990.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.38461538461538464, 1.6666666666666665))</td></tr><tr><td>10002</td><td>Engineer</td><td>1954-11-18</td><td>Byong</td><td>Delgrande</td><td>F</td><td>1991-09-07</td><td>65828</td><td>Sales</td><td>2.0</td><td>1.0</td><td>2.0</td><td>1954</td><td>1991</td><td>Map(vectorType -> dense, length -> 2, values -> List(1954.0, 1991.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.7692307692307693, 2.0))</td></tr><tr><td>10003</td><td>Engineer</td><td>1958-01-30</td><td>Berry</td><td>Babb</td><td>F</td><td>1992-03-21</td><td>40006</td><td>Production</td><td>2.0</td><td>1.0</td><td>1.0</td><td>1958</td><td>1992</td><td>Map(vectorType -> dense, length -> 2, values -> List(1958.0, 1992.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(2.307692307692308, 2.333333333333333))</td></tr><tr><td>10004</td><td>Staff</td><td>1957-09-28</td><td>Xiong</td><td>Verhoeff</td><td>M</td><td>1987-11-26</td><td>40054</td><td>Production</td><td>0.0</td><td>0.0</td><td>1.0</td><td>1957</td><td>1987</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1987.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 0.6666666666666666))</td></tr><tr><td>10005</td><td>Senior Engineer</td><td>1952-10-28</td><td>Abdelkader</td><td>Baumann</td><td>F</td><td>1991-01-18</td><td>78228</td><td>Human Resources</td><td>1.0</td><td>1.0</td><td>7.0</td><td>1952</td><td>1991</td><td>Map(vectorType -> dense, length -> 2, values -> List(1952.0, 1991.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.0, 2.0))</td></tr><tr><td>10006</td><td>Senior Engineer</td><td>1959-10-30</td><td>Eran</td><td>Cusworth</td><td>M</td><td>1986-11-14</td><td>40000</td><td>Development</td><td>1.0</td><td>0.0</td><td>0.0</td><td>1959</td><td>1986</td><td>Map(vectorType -> dense, length -> 2, values -> List(1959.0, 1986.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(2.6923076923076925, 0.3333333333333333))</td></tr><tr><td>10007</td><td>Staff</td><td>1957-04-14</td><td>Christoph</td><td>Parfitt</td><td>M</td><td>1991-06-28</td><td>56724</td><td>Research</td><td>0.0</td><td>0.0</td><td>4.0</td><td>1957</td><td>1991</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1991.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 2.0))</td></tr><tr><td>10008</td><td>Senior Staff</td><td>1964-11-17</td><td>Xudong</td><td>Samarati</td><td>M</td><td>1985-11-13</td><td>46671</td><td>Development</td><td>3.0</td><td>0.0</td><td>0.0</td><td>1964</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1964.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(4.615384615384616, 0.0))</td></tr><tr><td>10009</td><td>Engineer</td><td>1962-12-18</td><td>Lihong</td><td>Magliocco</td><td>M</td><td>1993-10-23</td><td>60929</td><td>Quality Management</td><td>2.0</td><td>0.0</td><td>6.0</td><td>1962</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1962.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(3.8461538461538463, 2.6666666666666665))</td></tr><tr><td>10010</td><td>Senior Staff</td><td>1956-04-24</td><td>Kwangyoen</td><td>Speek</td><td>F</td><td>1993-02-14</td><td>72488</td><td>Production</td><td>3.0</td><td>1.0</td><td>1.0</td><td>1956</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1956.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.5384615384615385, 2.6666666666666665))</td></tr><tr><td>10010</td><td>Senior Staff</td><td>1956-04-24</td><td>Kwangyoen</td><td>Speek</td><td>F</td><td>1993-02-14</td><td>72488</td><td>Quality Management</td><td>3.0</td><td>1.0</td><td>6.0</td><td>1956</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1956.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.5384615384615385, 2.6666666666666665))</td></tr><tr><td>10011</td><td>Engineer</td><td>1961-12-31</td><td>Shuichi</td><td>Tyugu</td><td>F</td><td>1995-01-17</td><td>42365</td><td>Customer Service</td><td>2.0</td><td>1.0</td><td>3.0</td><td>1961</td><td>1995</td><td>Map(vectorType -> dense, length -> 2, values -> List(1961.0, 1995.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(3.4615384615384617, 3.333333333333333))</td></tr><tr><td>10012</td><td>Senior Engineer</td><td>1961-10-25</td><td>Abdulah</td><td>Lunn</td><td>M</td><td>1989-04-08</td><td>40000</td><td>Development</td><td>1.0</td><td>0.0</td><td>0.0</td><td>1961</td><td>1989</td><td>Map(vectorType -> dense, length -> 2, values -> List(1961.0, 1989.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(3.4615384615384617, 1.3333333333333333))</td></tr><tr><td>10013</td><td>Senior Engineer</td><td>1961-08-30</td><td>Perry</td><td>Lorho</td><td>F</td><td>1991-08-03</td><td>40000</td><td>Human Resources</td><td>1.0</td><td>1.0</td><td>7.0</td><td>1961</td><td>1991</td><td>Map(vectorType -> dense, length -> 2, values -> List(1961.0, 1991.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(3.4615384615384617, 2.0))</td></tr><tr><td>10014</td><td>Staff</td><td>1958-03-23</td><td>Subbu</td><td>Bultermann</td><td>F</td><td>1988-03-25</td><td>46168</td><td>Development</td><td>0.0</td><td>1.0</td><td>0.0</td><td>1958</td><td>1988</td><td>Map(vectorType -> dense, length -> 2, values -> List(1958.0, 1988.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(2.307692307692308, 1.0))</td></tr><tr><td>10015</td><td>Engineer</td><td>1959-05-19</td><td>Bojan</td><td>Zallocco</td><td>M</td><td>1986-10-14</td><td>40000</td><td>Research</td><td>2.0</td><td>0.0</td><td>4.0</td><td>1959</td><td>1986</td><td>Map(vectorType -> dense, length -> 2, values -> List(1959.0, 1986.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(2.6923076923076925, 0.3333333333333333))</td></tr><tr><td>10016</td><td>Technique Leader</td><td>1963-07-12</td><td>Bilhanan</td><td>Wuwongse</td><td>M</td><td>1993-10-06</td><td>70889</td><td>Sales</td><td>4.0</td><td>0.0</td><td>2.0</td><td>1963</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1963.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(4.230769230769231, 2.6666666666666665))</td></tr><tr><td>10017</td><td>Staff</td><td>1964-01-19</td><td>Venkatesan</td><td>Gilg</td><td>M</td><td>1993-06-28</td><td>71380</td><td>Marketing</td><td>0.0</td><td>0.0</td><td>5.0</td><td>1964</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1964.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(4.615384615384616, 2.6666666666666665))</td></tr><tr><td>10018</td><td>Engineer</td><td>1957-12-11</td><td>Naftali</td><td>Dulli</td><td>M</td><td>1993-06-06</td><td>55881</td><td>Production</td><td>2.0</td><td>0.0</td><td>1.0</td><td>1957</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 2.6666666666666665))</td></tr><tr><td>10018</td><td>Engineer</td><td>1957-12-11</td><td>Naftali</td><td>Dulli</td><td>M</td><td>1993-06-06</td><td>55881</td><td>Development</td><td>2.0</td><td>0.0</td><td>0.0</td><td>1957</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 2.6666666666666665))</td></tr></tbody></table></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "aggData": [],
       "aggError": "",
       "aggOverflow": false,
       "aggSchema": [],
       "aggSeriesLimitReached": false,
       "aggType": "",
       "arguments": {},
       "columnCustomDisplayInfos": {},
       "data": [
        [
         10001,
         "Staff",
         "1953-07-25",
         "Hideyuki",
         "Zallocco",
         "M",
         "1990-04-28",
         60117,
         "Development",
         0.0,
         0.0,
         0.0,
         1953,
         1990,
         {
          "length": 2,
          "values": [
           1953.0,
           1990.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.38461538461538464,
           1.6666666666666665
          ],
          "vectorType": "dense"
         }
        ],
        [
         10002,
         "Engineer",
         "1954-11-18",
         "Byong",
         "Delgrande",
         "F",
         "1991-09-07",
         65828,
         "Sales",
         2.0,
         1.0,
         2.0,
         1954,
         1991,
         {
          "length": 2,
          "values": [
           1954.0,
           1991.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.7692307692307693,
           2.0
          ],
          "vectorType": "dense"
         }
        ],
        [
         10003,
         "Engineer",
         "1958-01-30",
         "Berry",
         "Babb",
         "F",
         "1992-03-21",
         40006,
         "Production",
         2.0,
         1.0,
         1.0,
         1958,
         1992,
         {
          "length": 2,
          "values": [
           1958.0,
           1992.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           2.307692307692308,
           2.333333333333333
          ],
          "vectorType": "dense"
         }
        ],
        [
         10004,
         "Staff",
         "1957-09-28",
         "Xiong",
         "Verhoeff",
         "M",
         "1987-11-26",
         40054,
         "Production",
         0.0,
         0.0,
         1.0,
         1957,
         1987,
         {
          "length": 2,
          "values": [
           1957.0,
           1987.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           0.6666666666666666
          ],
          "vectorType": "dense"
         }
        ],
        [
         10005,
         "Senior Engineer",
         "1952-10-28",
         "Abdelkader",
         "Baumann",
         "F",
         "1991-01-18",
         78228,
         "Human Resources",
         1.0,
         1.0,
         7.0,
         1952,
         1991,
         {
          "length": 2,
          "values": [
           1952.0,
           1991.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.0,
           2.0
          ],
          "vectorType": "dense"
         }
        ],
        [
         10006,
         "Senior Engineer",
         "1959-10-30",
         "Eran",
         "Cusworth",
         "M",
         "1986-11-14",
         40000,
         "Development",
         1.0,
         0.0,
         0.0,
         1959,
         1986,
         {
          "length": 2,
          "values": [
           1959.0,
           1986.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           2.6923076923076925,
           0.3333333333333333
          ],
          "vectorType": "dense"
         }
        ],
        [
         10007,
         "Staff",
         "1957-04-14",
         "Christoph",
         "Parfitt",
         "M",
         "1991-06-28",
         56724,
         "Research",
         0.0,
         0.0,
         4.0,
         1957,
         1991,
         {
          "length": 2,
          "values": [
           1957.0,
           1991.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           2.0
          ],
          "vectorType": "dense"
         }
        ],
        [
         10008,
         "Senior Staff",
         "1964-11-17",
         "Xudong",
         "Samarati",
         "M",
         "1985-11-13",
         46671,
         "Development",
         3.0,
         0.0,
         0.0,
         1964,
         1985,
         {
          "length": 2,
          "values": [
           1964.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           4.615384615384616,
           0.0
          ],
          "vectorType": "dense"
         }
        ],
        [
         10009,
         "Engineer",
         "1962-12-18",
         "Lihong",
         "Magliocco",
         "M",
         "1993-10-23",
         60929,
         "Quality Management",
         2.0,
         0.0,
         6.0,
         1962,
         1993,
         {
          "length": 2,
          "values": [
           1962.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           3.8461538461538463,
           2.6666666666666665
          ],
          "vectorType": "dense"
         }
        ],
        [
         10010,
         "Senior Staff",
         "1956-04-24",
         "Kwangyoen",
         "Speek",
         "F",
         "1993-02-14",
         72488,
         "Production",
         3.0,
         1.0,
         1.0,
         1956,
         1993,
         {
          "length": 2,
          "values": [
           1956.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.5384615384615385,
           2.6666666666666665
          ],
          "vectorType": "dense"
         }
        ],
        [
         10010,
         "Senior Staff",
         "1956-04-24",
         "Kwangyoen",
         "Speek",
         "F",
         "1993-02-14",
         72488,
         "Quality Management",
         3.0,
         1.0,
         6.0,
         1956,
         1993,
         {
          "length": 2,
          "values": [
           1956.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.5384615384615385,
           2.6666666666666665
          ],
          "vectorType": "dense"
         }
        ],
        [
         10011,
         "Engineer",
         "1961-12-31",
         "Shuichi",
         "Tyugu",
         "F",
         "1995-01-17",
         42365,
         "Customer Service",
         2.0,
         1.0,
         3.0,
         1961,
         1995,
         {
          "length": 2,
          "values": [
           1961.0,
           1995.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           3.4615384615384617,
           3.333333333333333
          ],
          "vectorType": "dense"
         }
        ],
        [
         10012,
         "Senior Engineer",
         "1961-10-25",
         "Abdulah",
         "Lunn",
         "M",
         "1989-04-08",
         40000,
         "Development",
         1.0,
         0.0,
         0.0,
         1961,
         1989,
         {
          "length": 2,
          "values": [
           1961.0,
           1989.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           3.4615384615384617,
           1.3333333333333333
          ],
          "vectorType": "dense"
         }
        ],
        [
         10013,
         "Senior Engineer",
         "1961-08-30",
         "Perry",
         "Lorho",
         "F",
         "1991-08-03",
         40000,
         "Human Resources",
         1.0,
         1.0,
         7.0,
         1961,
         1991,
         {
          "length": 2,
          "values": [
           1961.0,
           1991.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           3.4615384615384617,
           2.0
          ],
          "vectorType": "dense"
         }
        ],
        [
         10014,
         "Staff",
         "1958-03-23",
         "Subbu",
         "Bultermann",
         "F",
         "1988-03-25",
         46168,
         "Development",
         0.0,
         1.0,
         0.0,
         1958,
         1988,
         {
          "length": 2,
          "values": [
           1958.0,
           1988.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           2.307692307692308,
           1.0
          ],
          "vectorType": "dense"
         }
        ],
        [
         10015,
         "Engineer",
         "1959-05-19",
         "Bojan",
         "Zallocco",
         "M",
         "1986-10-14",
         40000,
         "Research",
         2.0,
         0.0,
         4.0,
         1959,
         1986,
         {
          "length": 2,
          "values": [
           1959.0,
           1986.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           2.6923076923076925,
           0.3333333333333333
          ],
          "vectorType": "dense"
         }
        ],
        [
         10016,
         "Technique Leader",
         "1963-07-12",
         "Bilhanan",
         "Wuwongse",
         "M",
         "1993-10-06",
         70889,
         "Sales",
         4.0,
         0.0,
         2.0,
         1963,
         1993,
         {
          "length": 2,
          "values": [
           1963.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           4.230769230769231,
           2.6666666666666665
          ],
          "vectorType": "dense"
         }
        ],
        [
         10017,
         "Staff",
         "1964-01-19",
         "Venkatesan",
         "Gilg",
         "M",
         "1993-06-28",
         71380,
         "Marketing",
         0.0,
         0.0,
         5.0,
         1964,
         1993,
         {
          "length": 2,
          "values": [
           1964.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           4.615384615384616,
           2.6666666666666665
          ],
          "vectorType": "dense"
         }
        ],
        [
         10018,
         "Engineer",
         "1957-12-11",
         "Naftali",
         "Dulli",
         "M",
         "1993-06-06",
         55881,
         "Production",
         2.0,
         0.0,
         1.0,
         1957,
         1993,
         {
          "length": 2,
          "values": [
           1957.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           2.6666666666666665
          ],
          "vectorType": "dense"
         }
        ],
        [
         10018,
         "Engineer",
         "1957-12-11",
         "Naftali",
         "Dulli",
         "M",
         "1993-06-06",
         55881,
         "Development",
         2.0,
         0.0,
         0.0,
         1957,
         1993,
         {
          "length": 2,
          "values": [
           1957.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           2.6666666666666665
          ],
          "vectorType": "dense"
         }
        ]
       ],
       "datasetInfos": [],
       "dbfsResultPath": null,
       "isJsonSchema": true,
       "metadata": {},
       "overflow": false,
       "plotOptions": {
        "customPlotOptions": {},
        "displayType": "table",
        "pivotAggregation": null,
        "pivotColumns": null,
        "xColumns": null,
        "yColumns": null
       },
       "removedWidgets": [],
       "schema": [
        {
         "metadata": "{}",
         "name": "emp_no",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "emp_level",
         "type": "\"string\""
        },
        {
         "metadata": "{\"__detected_date_formats\":\"yyyy-M-d\"}",
         "name": "birth_date",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "first_name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "last_name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "sex",
         "type": "\"string\""
        },
        {
         "metadata": "{\"__detected_date_formats\":\"yyyy-M-d\"}",
         "name": "hire_date",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "salary",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "department",
         "type": "\"string\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"Staff\",\"Senior Engineer\",\"Engineer\",\"Senior Staff\",\"Technique Leader\",\"Assistant Engineer\",\"Manager\"],\"type\":\"nominal\",\"name\":\"level_indexed\"}}",
         "name": "level_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"M\",\"F\"],\"type\":\"nominal\",\"name\":\"sex_indexed\"}}",
         "name": "sex_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"Development\",\"Production\",\"Sales\",\"Customer Service\",\"Research\",\"Marketing\",\"Quality Management\",\"Human Resources\",\"Finance\"],\"type\":\"nominal\",\"name\":\"dept_indexed\"}}",
         "name": "dept_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{}",
         "name": "birth_year",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "hire_year",
         "type": "\"integer\""
        },
        {
         "metadata": "{\"ml_attr\":{\"attrs\":{\"numeric\":[{\"idx\":0,\"name\":\"birth_year\"},{\"idx\":1,\"name\":\"hire_year\"}]},\"num_attrs\":2}}",
         "name": "year_union",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        },
        {
         "metadata": "{\"ml_attr\":{\"num_attrs\":2}}",
         "name": "scaled_year",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        }
       ],
       "type": "table"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pyspark.ml.feature import MinMaxScaler\n",
    "from pyspark.ml.functions import vector_to_array\n",
    "#apply MinMax Scaler for year figures\n",
    "\n",
    "year_assembler = VectorAssembler(inputCols =[\"birth_year\", \"hire_year\"], outputCol='year_union')\n",
    "emp_df = year_assembler.transform(emp_df)\n",
    "\n",
    "sc = MinMaxScaler(max=5.0, inputCol='year_union', outputCol = 'scaled_year')\n",
    "emp_df = sc.fit(emp_df).transform(emp_df)\n",
    "\n",
    "display(emp_df.limit(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "a51170f5-25e7-424a-b5ab-2e1cd099fd60",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "emp_df = emp_df.withColumn(\n",
    "    \"scaled_birth\",\n",
    "    vector_to_array(col(\"scaled_year\"))[0]\n",
    ").withColumn(\n",
    "    \"scaled_hire\",\n",
    "    vector_to_array(col(\"scaled_year\"))[0]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "b1811add-ec45-48f0-a080-59be1c773fb9",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Out[99]: ['emp_no',\n 'emp_level',\n 'birth_date',\n 'first_name',\n 'last_name',\n 'sex',\n 'hire_date',\n 'salary',\n 'department',\n 'level_indexed',\n 'sex_indexed',\n 'dept_indexed',\n 'birth_year',\n 'hire_year',\n 'year_union',\n 'scaled_year',\n 'scaled_birth',\n 'scaled_hire']"
     ]
    }
   ],
   "source": [
    "emp_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "feba5d3b-f972-48b0-a9ec-0000650360bb",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "#assemble all the prediction features\n",
    "\n",
    "assembler = VectorAssembler(inputCols = ['level_indexed', \"sex_indexed\", \"dept_indexed\", \"scaled_birth\", \"scaled_hire\"],\n",
    "                            outputCol = \"predictors\")\n",
    "\n",
    "\n",
    "emp_df = assembler.transform(emp_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "2651f1fa-efaa-4a3b-a351-e40aa6408ae9",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape before duplicates dropped [331603, 19]\n"
     ]
    }
   ],
   "source": [
    "emp_df = emp_df.sort([\"hire_date\",\"birth_date\"], asc=False)  #sort the data by hire date and birth date\n",
    "\n",
    "print(\"Shape before duplicates dropped\", shape(emp_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "2badcb9c-26a9-41f6-a2c9-af847aebd668",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>emp_no</th><th>emp_level</th><th>birth_date</th><th>first_name</th><th>last_name</th><th>sex</th><th>hire_date</th><th>salary</th><th>department</th><th>level_indexed</th><th>sex_indexed</th><th>dept_indexed</th><th>birth_year</th><th>hire_year</th><th>year_union</th><th>scaled_year</th><th>scaled_birth</th><th>scaled_hire</th><th>predictors</th></tr></thead><tbody><tr><td>217245</td><td>Manager</td><td>1953-06-24</td><td>Shirish</td><td>Ossenbruggen</td><td>F</td><td>1985-01-01</td><td>68318</td><td>Finance</td><td>6.0</td><td>1.0</td><td>8.0</td><td>1953</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1953.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.38461538461538464, 0.0))</td><td>0.38461538461538464</td><td>0.38461538461538464</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 1.0, 8.0, 0.38461538461538464, 0.38461538461538464))</td></tr><tr><td>415921</td><td>Manager</td><td>1954-10-05</td><td>Tonny</td><td>Butterworth</td><td>F</td><td>1985-01-01</td><td>40000</td><td>Research</td><td>6.0</td><td>1.0</td><td>4.0</td><td>1954</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1954.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.7692307692307693, 0.0))</td><td>0.7692307692307693</td><td>0.7692307692307693</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 1.0, 4.0, 0.7692307692307693, 0.7692307692307693))</td></tr><tr><td>276153</td><td>Manager</td><td>1956-06-08</td><td>Krassimir</td><td>Wegerle</td><td>F</td><td>1985-01-01</td><td>41491</td><td>Finance</td><td>6.0</td><td>1.0</td><td>8.0</td><td>1956</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1956.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.5384615384615385, 0.0))</td><td>1.5384615384615385</td><td>1.5384615384615385</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 1.0, 8.0, 1.5384615384615385, 1.5384615384615385))</td></tr><tr><td>276153</td><td>Manager</td><td>1956-06-08</td><td>Krassimir</td><td>Wegerle</td><td>F</td><td>1985-01-01</td><td>41491</td><td>Sales</td><td>6.0</td><td>1.0</td><td>2.0</td><td>1956</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1956.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.5384615384615385, 0.0))</td><td>1.5384615384615385</td><td>1.5384615384615385</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 1.0, 2.0, 1.5384615384615385, 1.5384615384615385))</td></tr><tr><td>69075</td><td>Manager</td><td>1956-09-12</td><td>Margareta</td><td>Markovitch</td><td>M</td><td>1985-01-01</td><td>44154</td><td>Finance</td><td>6.0</td><td>0.0</td><td>8.0</td><td>1956</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1956.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.5384615384615385, 0.0))</td><td>1.5384615384615385</td><td>1.5384615384615385</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 8.0, 1.5384615384615385, 1.5384615384615385))</td></tr><tr><td>69075</td><td>Manager</td><td>1956-09-12</td><td>Margareta</td><td>Markovitch</td><td>M</td><td>1985-01-01</td><td>44154</td><td>Human Resources</td><td>6.0</td><td>0.0</td><td>7.0</td><td>1956</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1956.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.5384615384615385, 0.0))</td><td>1.5384615384615385</td><td>1.5384615384615385</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 7.0, 1.5384615384615385, 1.5384615384615385))</td></tr><tr><td>94810</td><td>Manager</td><td>1957-07-08</td><td>DeForest</td><td>Hagimont</td><td>M</td><td>1985-01-01</td><td>64508</td><td>Production</td><td>6.0</td><td>0.0</td><td>1.0</td><td>1957</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 0.0))</td><td>1.9230769230769231</td><td>1.9230769230769231</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 1.0, 1.9230769230769231, 1.9230769230769231))</td></tr><tr><td>94810</td><td>Manager</td><td>1957-07-08</td><td>DeForest</td><td>Hagimont</td><td>M</td><td>1985-01-01</td><td>64508</td><td>Development</td><td>6.0</td><td>0.0</td><td>0.0</td><td>1957</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 0.0))</td><td>1.9230769230769231</td><td>1.9230769230769231</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 0.0, 1.9230769230769231, 1.9230769230769231))</td></tr><tr><td>469466</td><td>Manager</td><td>1959-10-28</td><td>Ebru</td><td>Alpin</td><td>M</td><td>1985-01-01</td><td>40000</td><td>Development</td><td>6.0</td><td>0.0</td><td>0.0</td><td>1959</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1959.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(2.6923076923076925, 0.0))</td><td>2.6923076923076925</td><td>2.6923076923076925</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 0.0, 2.6923076923076925, 2.6923076923076925))</td></tr><tr><td>424772</td><td>Manager</td><td>1959-11-09</td><td>Arie</td><td>Staelin</td><td>M</td><td>1985-01-01</td><td>48747</td><td>Development</td><td>6.0</td><td>0.0</td><td>0.0</td><td>1959</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1959.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(2.6923076923076925, 0.0))</td><td>2.6923076923076925</td><td>2.6923076923076925</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 0.0, 2.6923076923076925, 2.6923076923076925))</td></tr><tr><td>54345</td><td>Manager</td><td>1961-03-14</td><td>Peternela</td><td>Onuegbe</td><td>F</td><td>1985-01-01</td><td>79216</td><td>Production</td><td>6.0</td><td>1.0</td><td>1.0</td><td>1961</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1961.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(3.4615384615384617, 0.0))</td><td>3.4615384615384617</td><td>3.4615384615384617</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 1.0, 1.0, 3.4615384615384617, 3.4615384615384617))</td></tr><tr><td>54345</td><td>Manager</td><td>1961-03-14</td><td>Peternela</td><td>Onuegbe</td><td>F</td><td>1985-01-01</td><td>79216</td><td>Development</td><td>6.0</td><td>1.0</td><td>0.0</td><td>1961</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1961.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(3.4615384615384617, 0.0))</td><td>3.4615384615384617</td><td>3.4615384615384617</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 1.0, 0.0, 3.4615384615384617, 3.4615384615384617))</td></tr><tr><td>71953</td><td>Manager</td><td>1962-02-24</td><td>Przemyslawa</td><td>Kaelbling</td><td>M</td><td>1985-01-01</td><td>40000</td><td>Customer Service</td><td>6.0</td><td>0.0</td><td>3.0</td><td>1962</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1962.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(3.8461538461538463, 0.0))</td><td>3.8461538461538463</td><td>3.8461538461538463</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 3.0, 3.8461538461538463, 3.8461538461538463))</td></tr><tr><td>237584</td><td>Manager</td><td>1957-03-28</td><td>Isamu</td><td>Legleitner</td><td>F</td><td>1985-01-14</td><td>75112</td><td>Sales</td><td>6.0</td><td>1.0</td><td>2.0</td><td>1957</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 0.0))</td><td>1.9230769230769231</td><td>1.9230769230769231</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 1.0, 2.0, 1.9230769230769231, 1.9230769230769231))</td></tr><tr><td>109525</td><td>Senior Engineer</td><td>1952-02-26</td><td>Mona</td><td>Beutelspacher</td><td>M</td><td>1985-02-01</td><td>51693</td><td>Development</td><td>1.0</td><td>0.0</td><td>0.0</td><td>1952</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1952.0, 1985.0))</td><td>Map(vectorType -> sparse, length -> 2, indices -> List(), values -> List())</td><td>0.0</td><td>0.0</td><td>Map(vectorType -> sparse, length -> 5, indices -> List(0), values -> List(1.0))</td></tr><tr><td>480930</td><td>Engineer</td><td>1952-06-29</td><td>Taizo</td><td>Orsini</td><td>M</td><td>1985-02-01</td><td>76549</td><td>Development</td><td>2.0</td><td>0.0</td><td>0.0</td><td>1952</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1952.0, 1985.0))</td><td>Map(vectorType -> sparse, length -> 2, indices -> List(), values -> List())</td><td>0.0</td><td>0.0</td><td>Map(vectorType -> sparse, length -> 5, indices -> List(0), values -> List(2.0))</td></tr><tr><td>253048</td><td>Senior Engineer</td><td>1952-12-15</td><td>Kannan</td><td>Mahnke</td><td>M</td><td>1985-02-01</td><td>40000</td><td>Human Resources</td><td>1.0</td><td>0.0</td><td>7.0</td><td>1952</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1952.0, 1985.0))</td><td>Map(vectorType -> sparse, length -> 2, indices -> List(), values -> List())</td><td>0.0</td><td>0.0</td><td>Map(vectorType -> sparse, length -> 5, indices -> List(0, 2), values -> List(1.0, 7.0))</td></tr><tr><td>452915</td><td>Senior Engineer</td><td>1954-11-24</td><td>Divier</td><td>Marrakchi</td><td>F</td><td>1985-02-01</td><td>55703</td><td>Production</td><td>1.0</td><td>1.0</td><td>1.0</td><td>1954</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1954.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.7692307692307693, 0.0))</td><td>0.7692307692307693</td><td>0.7692307692307693</td><td>Map(vectorType -> dense, length -> 5, values -> List(1.0, 1.0, 1.0, 0.7692307692307693, 0.7692307692307693))</td></tr><tr><td>77164</td><td>Senior Engineer</td><td>1955-09-05</td><td>Zdislav</td><td>Ferriere</td><td>F</td><td>1985-02-01</td><td>75052</td><td>Sales</td><td>1.0</td><td>1.0</td><td>2.0</td><td>1955</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1955.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.153846153846154, 0.0))</td><td>1.153846153846154</td><td>1.153846153846154</td><td>Map(vectorType -> dense, length -> 5, values -> List(1.0, 1.0, 2.0, 1.153846153846154, 1.153846153846154))</td></tr><tr><td>469585</td><td>Staff</td><td>1955-09-17</td><td>Yongdong</td><td>Pileggi</td><td>M</td><td>1985-02-01</td><td>91063</td><td>Marketing</td><td>0.0</td><td>0.0</td><td>5.0</td><td>1955</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1955.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.153846153846154, 0.0))</td><td>1.153846153846154</td><td>1.153846153846154</td><td>Map(vectorType -> dense, length -> 5, values -> List(0.0, 0.0, 5.0, 1.153846153846154, 1.153846153846154))</td></tr></tbody></table></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "aggData": [],
       "aggError": "",
       "aggOverflow": false,
       "aggSchema": [],
       "aggSeriesLimitReached": false,
       "aggType": "",
       "arguments": {},
       "columnCustomDisplayInfos": {},
       "data": [
        [
         217245,
         "Manager",
         "1953-06-24",
         "Shirish",
         "Ossenbruggen",
         "F",
         "1985-01-01",
         68318,
         "Finance",
         6.0,
         1.0,
         8.0,
         1953,
         1985,
         {
          "length": 2,
          "values": [
           1953.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.38461538461538464,
           0.0
          ],
          "vectorType": "dense"
         },
         0.38461538461538464,
         0.38461538461538464,
         {
          "length": 5,
          "values": [
           6.0,
           1.0,
           8.0,
           0.38461538461538464,
           0.38461538461538464
          ],
          "vectorType": "dense"
         }
        ],
        [
         415921,
         "Manager",
         "1954-10-05",
         "Tonny",
         "Butterworth",
         "F",
         "1985-01-01",
         40000,
         "Research",
         6.0,
         1.0,
         4.0,
         1954,
         1985,
         {
          "length": 2,
          "values": [
           1954.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.7692307692307693,
           0.0
          ],
          "vectorType": "dense"
         },
         0.7692307692307693,
         0.7692307692307693,
         {
          "length": 5,
          "values": [
           6.0,
           1.0,
           4.0,
           0.7692307692307693,
           0.7692307692307693
          ],
          "vectorType": "dense"
         }
        ],
        [
         276153,
         "Manager",
         "1956-06-08",
         "Krassimir",
         "Wegerle",
         "F",
         "1985-01-01",
         41491,
         "Finance",
         6.0,
         1.0,
         8.0,
         1956,
         1985,
         {
          "length": 2,
          "values": [
           1956.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.5384615384615385,
           0.0
          ],
          "vectorType": "dense"
         },
         1.5384615384615385,
         1.5384615384615385,
         {
          "length": 5,
          "values": [
           6.0,
           1.0,
           8.0,
           1.5384615384615385,
           1.5384615384615385
          ],
          "vectorType": "dense"
         }
        ],
        [
         276153,
         "Manager",
         "1956-06-08",
         "Krassimir",
         "Wegerle",
         "F",
         "1985-01-01",
         41491,
         "Sales",
         6.0,
         1.0,
         2.0,
         1956,
         1985,
         {
          "length": 2,
          "values": [
           1956.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.5384615384615385,
           0.0
          ],
          "vectorType": "dense"
         },
         1.5384615384615385,
         1.5384615384615385,
         {
          "length": 5,
          "values": [
           6.0,
           1.0,
           2.0,
           1.5384615384615385,
           1.5384615384615385
          ],
          "vectorType": "dense"
         }
        ],
        [
         69075,
         "Manager",
         "1956-09-12",
         "Margareta",
         "Markovitch",
         "M",
         "1985-01-01",
         44154,
         "Finance",
         6.0,
         0.0,
         8.0,
         1956,
         1985,
         {
          "length": 2,
          "values": [
           1956.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.5384615384615385,
           0.0
          ],
          "vectorType": "dense"
         },
         1.5384615384615385,
         1.5384615384615385,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           8.0,
           1.5384615384615385,
           1.5384615384615385
          ],
          "vectorType": "dense"
         }
        ],
        [
         69075,
         "Manager",
         "1956-09-12",
         "Margareta",
         "Markovitch",
         "M",
         "1985-01-01",
         44154,
         "Human Resources",
         6.0,
         0.0,
         7.0,
         1956,
         1985,
         {
          "length": 2,
          "values": [
           1956.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.5384615384615385,
           0.0
          ],
          "vectorType": "dense"
         },
         1.5384615384615385,
         1.5384615384615385,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           7.0,
           1.5384615384615385,
           1.5384615384615385
          ],
          "vectorType": "dense"
         }
        ],
        [
         94810,
         "Manager",
         "1957-07-08",
         "DeForest",
         "Hagimont",
         "M",
         "1985-01-01",
         64508,
         "Production",
         6.0,
         0.0,
         1.0,
         1957,
         1985,
         {
          "length": 2,
          "values": [
           1957.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           0.0
          ],
          "vectorType": "dense"
         },
         1.9230769230769231,
         1.9230769230769231,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           1.0,
           1.9230769230769231,
           1.9230769230769231
          ],
          "vectorType": "dense"
         }
        ],
        [
         94810,
         "Manager",
         "1957-07-08",
         "DeForest",
         "Hagimont",
         "M",
         "1985-01-01",
         64508,
         "Development",
         6.0,
         0.0,
         0.0,
         1957,
         1985,
         {
          "length": 2,
          "values": [
           1957.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           0.0
          ],
          "vectorType": "dense"
         },
         1.9230769230769231,
         1.9230769230769231,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           0.0,
           1.9230769230769231,
           1.9230769230769231
          ],
          "vectorType": "dense"
         }
        ],
        [
         469466,
         "Manager",
         "1959-10-28",
         "Ebru",
         "Alpin",
         "M",
         "1985-01-01",
         40000,
         "Development",
         6.0,
         0.0,
         0.0,
         1959,
         1985,
         {
          "length": 2,
          "values": [
           1959.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           2.6923076923076925,
           0.0
          ],
          "vectorType": "dense"
         },
         2.6923076923076925,
         2.6923076923076925,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           0.0,
           2.6923076923076925,
           2.6923076923076925
          ],
          "vectorType": "dense"
         }
        ],
        [
         424772,
         "Manager",
         "1959-11-09",
         "Arie",
         "Staelin",
         "M",
         "1985-01-01",
         48747,
         "Development",
         6.0,
         0.0,
         0.0,
         1959,
         1985,
         {
          "length": 2,
          "values": [
           1959.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           2.6923076923076925,
           0.0
          ],
          "vectorType": "dense"
         },
         2.6923076923076925,
         2.6923076923076925,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           0.0,
           2.6923076923076925,
           2.6923076923076925
          ],
          "vectorType": "dense"
         }
        ],
        [
         54345,
         "Manager",
         "1961-03-14",
         "Peternela",
         "Onuegbe",
         "F",
         "1985-01-01",
         79216,
         "Production",
         6.0,
         1.0,
         1.0,
         1961,
         1985,
         {
          "length": 2,
          "values": [
           1961.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           3.4615384615384617,
           0.0
          ],
          "vectorType": "dense"
         },
         3.4615384615384617,
         3.4615384615384617,
         {
          "length": 5,
          "values": [
           6.0,
           1.0,
           1.0,
           3.4615384615384617,
           3.4615384615384617
          ],
          "vectorType": "dense"
         }
        ],
        [
         54345,
         "Manager",
         "1961-03-14",
         "Peternela",
         "Onuegbe",
         "F",
         "1985-01-01",
         79216,
         "Development",
         6.0,
         1.0,
         0.0,
         1961,
         1985,
         {
          "length": 2,
          "values": [
           1961.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           3.4615384615384617,
           0.0
          ],
          "vectorType": "dense"
         },
         3.4615384615384617,
         3.4615384615384617,
         {
          "length": 5,
          "values": [
           6.0,
           1.0,
           0.0,
           3.4615384615384617,
           3.4615384615384617
          ],
          "vectorType": "dense"
         }
        ],
        [
         71953,
         "Manager",
         "1962-02-24",
         "Przemyslawa",
         "Kaelbling",
         "M",
         "1985-01-01",
         40000,
         "Customer Service",
         6.0,
         0.0,
         3.0,
         1962,
         1985,
         {
          "length": 2,
          "values": [
           1962.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           3.8461538461538463,
           0.0
          ],
          "vectorType": "dense"
         },
         3.8461538461538463,
         3.8461538461538463,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           3.0,
           3.8461538461538463,
           3.8461538461538463
          ],
          "vectorType": "dense"
         }
        ],
        [
         237584,
         "Manager",
         "1957-03-28",
         "Isamu",
         "Legleitner",
         "F",
         "1985-01-14",
         75112,
         "Sales",
         6.0,
         1.0,
         2.0,
         1957,
         1985,
         {
          "length": 2,
          "values": [
           1957.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           0.0
          ],
          "vectorType": "dense"
         },
         1.9230769230769231,
         1.9230769230769231,
         {
          "length": 5,
          "values": [
           6.0,
           1.0,
           2.0,
           1.9230769230769231,
           1.9230769230769231
          ],
          "vectorType": "dense"
         }
        ],
        [
         109525,
         "Senior Engineer",
         "1952-02-26",
         "Mona",
         "Beutelspacher",
         "M",
         "1985-02-01",
         51693,
         "Development",
         1.0,
         0.0,
         0.0,
         1952,
         1985,
         {
          "length": 2,
          "values": [
           1952.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "indices": [],
          "length": 2,
          "values": [],
          "vectorType": "sparse"
         },
         0.0,
         0.0,
         {
          "indices": [
           0
          ],
          "length": 5,
          "values": [
           1.0
          ],
          "vectorType": "sparse"
         }
        ],
        [
         480930,
         "Engineer",
         "1952-06-29",
         "Taizo",
         "Orsini",
         "M",
         "1985-02-01",
         76549,
         "Development",
         2.0,
         0.0,
         0.0,
         1952,
         1985,
         {
          "length": 2,
          "values": [
           1952.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "indices": [],
          "length": 2,
          "values": [],
          "vectorType": "sparse"
         },
         0.0,
         0.0,
         {
          "indices": [
           0
          ],
          "length": 5,
          "values": [
           2.0
          ],
          "vectorType": "sparse"
         }
        ],
        [
         253048,
         "Senior Engineer",
         "1952-12-15",
         "Kannan",
         "Mahnke",
         "M",
         "1985-02-01",
         40000,
         "Human Resources",
         1.0,
         0.0,
         7.0,
         1952,
         1985,
         {
          "length": 2,
          "values": [
           1952.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "indices": [],
          "length": 2,
          "values": [],
          "vectorType": "sparse"
         },
         0.0,
         0.0,
         {
          "indices": [
           0,
           2
          ],
          "length": 5,
          "values": [
           1.0,
           7.0
          ],
          "vectorType": "sparse"
         }
        ],
        [
         452915,
         "Senior Engineer",
         "1954-11-24",
         "Divier",
         "Marrakchi",
         "F",
         "1985-02-01",
         55703,
         "Production",
         1.0,
         1.0,
         1.0,
         1954,
         1985,
         {
          "length": 2,
          "values": [
           1954.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.7692307692307693,
           0.0
          ],
          "vectorType": "dense"
         },
         0.7692307692307693,
         0.7692307692307693,
         {
          "length": 5,
          "values": [
           1.0,
           1.0,
           1.0,
           0.7692307692307693,
           0.7692307692307693
          ],
          "vectorType": "dense"
         }
        ],
        [
         77164,
         "Senior Engineer",
         "1955-09-05",
         "Zdislav",
         "Ferriere",
         "F",
         "1985-02-01",
         75052,
         "Sales",
         1.0,
         1.0,
         2.0,
         1955,
         1985,
         {
          "length": 2,
          "values": [
           1955.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.153846153846154,
           0.0
          ],
          "vectorType": "dense"
         },
         1.153846153846154,
         1.153846153846154,
         {
          "length": 5,
          "values": [
           1.0,
           1.0,
           2.0,
           1.153846153846154,
           1.153846153846154
          ],
          "vectorType": "dense"
         }
        ],
        [
         469585,
         "Staff",
         "1955-09-17",
         "Yongdong",
         "Pileggi",
         "M",
         "1985-02-01",
         91063,
         "Marketing",
         0.0,
         0.0,
         5.0,
         1955,
         1985,
         {
          "length": 2,
          "values": [
           1955.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.153846153846154,
           0.0
          ],
          "vectorType": "dense"
         },
         1.153846153846154,
         1.153846153846154,
         {
          "length": 5,
          "values": [
           0.0,
           0.0,
           5.0,
           1.153846153846154,
           1.153846153846154
          ],
          "vectorType": "dense"
         }
        ]
       ],
       "datasetInfos": [],
       "dbfsResultPath": null,
       "isJsonSchema": true,
       "metadata": {},
       "overflow": false,
       "plotOptions": {
        "customPlotOptions": {},
        "displayType": "table",
        "pivotAggregation": null,
        "pivotColumns": null,
        "xColumns": null,
        "yColumns": null
       },
       "removedWidgets": [],
       "schema": [
        {
         "metadata": "{}",
         "name": "emp_no",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "emp_level",
         "type": "\"string\""
        },
        {
         "metadata": "{\"__detected_date_formats\":\"yyyy-M-d\"}",
         "name": "birth_date",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "first_name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "last_name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "sex",
         "type": "\"string\""
        },
        {
         "metadata": "{\"__detected_date_formats\":\"yyyy-M-d\"}",
         "name": "hire_date",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "salary",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "department",
         "type": "\"string\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"Staff\",\"Senior Engineer\",\"Engineer\",\"Senior Staff\",\"Technique Leader\",\"Assistant Engineer\",\"Manager\"],\"type\":\"nominal\",\"name\":\"level_indexed\"}}",
         "name": "level_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"M\",\"F\"],\"type\":\"nominal\",\"name\":\"sex_indexed\"}}",
         "name": "sex_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"Development\",\"Production\",\"Sales\",\"Customer Service\",\"Research\",\"Marketing\",\"Quality Management\",\"Human Resources\",\"Finance\"],\"type\":\"nominal\",\"name\":\"dept_indexed\"}}",
         "name": "dept_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{}",
         "name": "birth_year",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "hire_year",
         "type": "\"integer\""
        },
        {
         "metadata": "{\"ml_attr\":{\"attrs\":{\"numeric\":[{\"idx\":0,\"name\":\"birth_year\"},{\"idx\":1,\"name\":\"hire_year\"}]},\"num_attrs\":2}}",
         "name": "year_union",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        },
        {
         "metadata": "{\"ml_attr\":{\"num_attrs\":2}}",
         "name": "scaled_year",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        },
        {
         "metadata": "{}",
         "name": "scaled_birth",
         "type": "\"double\""
        },
        {
         "metadata": "{}",
         "name": "scaled_hire",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"attrs\":{\"numeric\":[{\"idx\":3,\"name\":\"scaled_birth\"},{\"idx\":4,\"name\":\"scaled_hire\"}],\"nominal\":[{\"vals\":[\"Staff\",\"Senior Engineer\",\"Engineer\",\"Senior Staff\",\"Technique Leader\",\"Assistant Engineer\",\"Manager\"],\"idx\":0,\"name\":\"level_indexed\"},{\"vals\":[\"M\",\"F\"],\"idx\":1,\"name\":\"sex_indexed\"},{\"vals\":[\"Development\",\"Production\",\"Sales\",\"Customer Service\",\"Research\",\"Marketing\",\"Quality Management\",\"Human Resources\",\"Finance\"],\"idx\":2,\"name\":\"dept_indexed\"}]},\"num_attrs\":5}}",
         "name": "predictors",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        }
       ],
       "type": "table"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(emp_df.sort([\"hire_date\",\"birth_date\"], asc=False).limit(20))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "fbfe9033-efab-4292-a2f0-97dfd64ed339",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>emp_no</th><th>emp_level</th><th>birth_date</th><th>first_name</th><th>last_name</th><th>sex</th><th>hire_date</th><th>salary</th><th>department</th><th>level_indexed</th><th>sex_indexed</th><th>dept_indexed</th><th>birth_year</th><th>hire_year</th><th>year_union</th><th>scaled_year</th><th>scaled_birth</th><th>scaled_hire</th><th>predictors</th></tr></thead><tbody><tr><td>217245</td><td>Manager</td><td>1953-06-24</td><td>Shirish</td><td>Ossenbruggen</td><td>F</td><td>1985-01-01</td><td>68318</td><td>Finance</td><td>6.0</td><td>1.0</td><td>8.0</td><td>1953</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1953.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.38461538461538464, 0.0))</td><td>0.38461538461538464</td><td>0.38461538461538464</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 1.0, 8.0, 0.38461538461538464, 0.38461538461538464))</td></tr><tr><td>415921</td><td>Manager</td><td>1954-10-05</td><td>Tonny</td><td>Butterworth</td><td>F</td><td>1985-01-01</td><td>40000</td><td>Research</td><td>6.0</td><td>1.0</td><td>4.0</td><td>1954</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1954.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.7692307692307693, 0.0))</td><td>0.7692307692307693</td><td>0.7692307692307693</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 1.0, 4.0, 0.7692307692307693, 0.7692307692307693))</td></tr><tr><td>276153</td><td>Manager</td><td>1956-06-08</td><td>Krassimir</td><td>Wegerle</td><td>F</td><td>1985-01-01</td><td>41491</td><td>Sales</td><td>6.0</td><td>1.0</td><td>2.0</td><td>1956</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1956.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.5384615384615385, 0.0))</td><td>1.5384615384615385</td><td>1.5384615384615385</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 1.0, 2.0, 1.5384615384615385, 1.5384615384615385))</td></tr><tr><td>276153</td><td>Manager</td><td>1956-06-08</td><td>Krassimir</td><td>Wegerle</td><td>F</td><td>1985-01-01</td><td>41491</td><td>Finance</td><td>6.0</td><td>1.0</td><td>8.0</td><td>1956</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1956.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.5384615384615385, 0.0))</td><td>1.5384615384615385</td><td>1.5384615384615385</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 1.0, 8.0, 1.5384615384615385, 1.5384615384615385))</td></tr><tr><td>69075</td><td>Manager</td><td>1956-09-12</td><td>Margareta</td><td>Markovitch</td><td>M</td><td>1985-01-01</td><td>44154</td><td>Human Resources</td><td>6.0</td><td>0.0</td><td>7.0</td><td>1956</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1956.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.5384615384615385, 0.0))</td><td>1.5384615384615385</td><td>1.5384615384615385</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 7.0, 1.5384615384615385, 1.5384615384615385))</td></tr><tr><td>69075</td><td>Manager</td><td>1956-09-12</td><td>Margareta</td><td>Markovitch</td><td>M</td><td>1985-01-01</td><td>44154</td><td>Finance</td><td>6.0</td><td>0.0</td><td>8.0</td><td>1956</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1956.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.5384615384615385, 0.0))</td><td>1.5384615384615385</td><td>1.5384615384615385</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 8.0, 1.5384615384615385, 1.5384615384615385))</td></tr><tr><td>94810</td><td>Manager</td><td>1957-07-08</td><td>DeForest</td><td>Hagimont</td><td>M</td><td>1985-01-01</td><td>64508</td><td>Production</td><td>6.0</td><td>0.0</td><td>1.0</td><td>1957</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 0.0))</td><td>1.9230769230769231</td><td>1.9230769230769231</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 1.0, 1.9230769230769231, 1.9230769230769231))</td></tr><tr><td>94810</td><td>Manager</td><td>1957-07-08</td><td>DeForest</td><td>Hagimont</td><td>M</td><td>1985-01-01</td><td>64508</td><td>Development</td><td>6.0</td><td>0.0</td><td>0.0</td><td>1957</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 0.0))</td><td>1.9230769230769231</td><td>1.9230769230769231</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 0.0, 1.9230769230769231, 1.9230769230769231))</td></tr><tr><td>469466</td><td>Manager</td><td>1959-10-28</td><td>Ebru</td><td>Alpin</td><td>M</td><td>1985-01-01</td><td>40000</td><td>Development</td><td>6.0</td><td>0.0</td><td>0.0</td><td>1959</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1959.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(2.6923076923076925, 0.0))</td><td>2.6923076923076925</td><td>2.6923076923076925</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 0.0, 2.6923076923076925, 2.6923076923076925))</td></tr><tr><td>424772</td><td>Manager</td><td>1959-11-09</td><td>Arie</td><td>Staelin</td><td>M</td><td>1985-01-01</td><td>48747</td><td>Development</td><td>6.0</td><td>0.0</td><td>0.0</td><td>1959</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1959.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(2.6923076923076925, 0.0))</td><td>2.6923076923076925</td><td>2.6923076923076925</td><td>Map(vectorType -> dense, length -> 5, values -> List(6.0, 0.0, 0.0, 2.6923076923076925, 2.6923076923076925))</td></tr></tbody></table></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "aggData": [],
       "aggError": "",
       "aggOverflow": false,
       "aggSchema": [],
       "aggSeriesLimitReached": false,
       "aggType": "",
       "arguments": {},
       "columnCustomDisplayInfos": {},
       "data": [
        [
         217245,
         "Manager",
         "1953-06-24",
         "Shirish",
         "Ossenbruggen",
         "F",
         "1985-01-01",
         68318,
         "Finance",
         6.0,
         1.0,
         8.0,
         1953,
         1985,
         {
          "length": 2,
          "values": [
           1953.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.38461538461538464,
           0.0
          ],
          "vectorType": "dense"
         },
         0.38461538461538464,
         0.38461538461538464,
         {
          "length": 5,
          "values": [
           6.0,
           1.0,
           8.0,
           0.38461538461538464,
           0.38461538461538464
          ],
          "vectorType": "dense"
         }
        ],
        [
         415921,
         "Manager",
         "1954-10-05",
         "Tonny",
         "Butterworth",
         "F",
         "1985-01-01",
         40000,
         "Research",
         6.0,
         1.0,
         4.0,
         1954,
         1985,
         {
          "length": 2,
          "values": [
           1954.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.7692307692307693,
           0.0
          ],
          "vectorType": "dense"
         },
         0.7692307692307693,
         0.7692307692307693,
         {
          "length": 5,
          "values": [
           6.0,
           1.0,
           4.0,
           0.7692307692307693,
           0.7692307692307693
          ],
          "vectorType": "dense"
         }
        ],
        [
         276153,
         "Manager",
         "1956-06-08",
         "Krassimir",
         "Wegerle",
         "F",
         "1985-01-01",
         41491,
         "Sales",
         6.0,
         1.0,
         2.0,
         1956,
         1985,
         {
          "length": 2,
          "values": [
           1956.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.5384615384615385,
           0.0
          ],
          "vectorType": "dense"
         },
         1.5384615384615385,
         1.5384615384615385,
         {
          "length": 5,
          "values": [
           6.0,
           1.0,
           2.0,
           1.5384615384615385,
           1.5384615384615385
          ],
          "vectorType": "dense"
         }
        ],
        [
         276153,
         "Manager",
         "1956-06-08",
         "Krassimir",
         "Wegerle",
         "F",
         "1985-01-01",
         41491,
         "Finance",
         6.0,
         1.0,
         8.0,
         1956,
         1985,
         {
          "length": 2,
          "values": [
           1956.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.5384615384615385,
           0.0
          ],
          "vectorType": "dense"
         },
         1.5384615384615385,
         1.5384615384615385,
         {
          "length": 5,
          "values": [
           6.0,
           1.0,
           8.0,
           1.5384615384615385,
           1.5384615384615385
          ],
          "vectorType": "dense"
         }
        ],
        [
         69075,
         "Manager",
         "1956-09-12",
         "Margareta",
         "Markovitch",
         "M",
         "1985-01-01",
         44154,
         "Human Resources",
         6.0,
         0.0,
         7.0,
         1956,
         1985,
         {
          "length": 2,
          "values": [
           1956.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.5384615384615385,
           0.0
          ],
          "vectorType": "dense"
         },
         1.5384615384615385,
         1.5384615384615385,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           7.0,
           1.5384615384615385,
           1.5384615384615385
          ],
          "vectorType": "dense"
         }
        ],
        [
         69075,
         "Manager",
         "1956-09-12",
         "Margareta",
         "Markovitch",
         "M",
         "1985-01-01",
         44154,
         "Finance",
         6.0,
         0.0,
         8.0,
         1956,
         1985,
         {
          "length": 2,
          "values": [
           1956.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.5384615384615385,
           0.0
          ],
          "vectorType": "dense"
         },
         1.5384615384615385,
         1.5384615384615385,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           8.0,
           1.5384615384615385,
           1.5384615384615385
          ],
          "vectorType": "dense"
         }
        ],
        [
         94810,
         "Manager",
         "1957-07-08",
         "DeForest",
         "Hagimont",
         "M",
         "1985-01-01",
         64508,
         "Production",
         6.0,
         0.0,
         1.0,
         1957,
         1985,
         {
          "length": 2,
          "values": [
           1957.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           0.0
          ],
          "vectorType": "dense"
         },
         1.9230769230769231,
         1.9230769230769231,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           1.0,
           1.9230769230769231,
           1.9230769230769231
          ],
          "vectorType": "dense"
         }
        ],
        [
         94810,
         "Manager",
         "1957-07-08",
         "DeForest",
         "Hagimont",
         "M",
         "1985-01-01",
         64508,
         "Development",
         6.0,
         0.0,
         0.0,
         1957,
         1985,
         {
          "length": 2,
          "values": [
           1957.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           0.0
          ],
          "vectorType": "dense"
         },
         1.9230769230769231,
         1.9230769230769231,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           0.0,
           1.9230769230769231,
           1.9230769230769231
          ],
          "vectorType": "dense"
         }
        ],
        [
         469466,
         "Manager",
         "1959-10-28",
         "Ebru",
         "Alpin",
         "M",
         "1985-01-01",
         40000,
         "Development",
         6.0,
         0.0,
         0.0,
         1959,
         1985,
         {
          "length": 2,
          "values": [
           1959.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           2.6923076923076925,
           0.0
          ],
          "vectorType": "dense"
         },
         2.6923076923076925,
         2.6923076923076925,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           0.0,
           2.6923076923076925,
           2.6923076923076925
          ],
          "vectorType": "dense"
         }
        ],
        [
         424772,
         "Manager",
         "1959-11-09",
         "Arie",
         "Staelin",
         "M",
         "1985-01-01",
         48747,
         "Development",
         6.0,
         0.0,
         0.0,
         1959,
         1985,
         {
          "length": 2,
          "values": [
           1959.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           2.6923076923076925,
           0.0
          ],
          "vectorType": "dense"
         },
         2.6923076923076925,
         2.6923076923076925,
         {
          "length": 5,
          "values": [
           6.0,
           0.0,
           0.0,
           2.6923076923076925,
           2.6923076923076925
          ],
          "vectorType": "dense"
         }
        ]
       ],
       "datasetInfos": [],
       "dbfsResultPath": null,
       "isJsonSchema": true,
       "metadata": {},
       "overflow": false,
       "plotOptions": {
        "customPlotOptions": {},
        "displayType": "table",
        "pivotAggregation": null,
        "pivotColumns": null,
        "xColumns": null,
        "yColumns": null
       },
       "removedWidgets": [],
       "schema": [
        {
         "metadata": "{}",
         "name": "emp_no",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "emp_level",
         "type": "\"string\""
        },
        {
         "metadata": "{\"__detected_date_formats\":\"yyyy-M-d\"}",
         "name": "birth_date",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "first_name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "last_name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "sex",
         "type": "\"string\""
        },
        {
         "metadata": "{\"__detected_date_formats\":\"yyyy-M-d\"}",
         "name": "hire_date",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "salary",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "department",
         "type": "\"string\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"Staff\",\"Senior Engineer\",\"Engineer\",\"Senior Staff\",\"Technique Leader\",\"Assistant Engineer\",\"Manager\"],\"type\":\"nominal\",\"name\":\"level_indexed\"}}",
         "name": "level_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"M\",\"F\"],\"type\":\"nominal\",\"name\":\"sex_indexed\"}}",
         "name": "sex_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"Development\",\"Production\",\"Sales\",\"Customer Service\",\"Research\",\"Marketing\",\"Quality Management\",\"Human Resources\",\"Finance\"],\"type\":\"nominal\",\"name\":\"dept_indexed\"}}",
         "name": "dept_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{}",
         "name": "birth_year",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "hire_year",
         "type": "\"integer\""
        },
        {
         "metadata": "{\"ml_attr\":{\"attrs\":{\"numeric\":[{\"idx\":0,\"name\":\"birth_year\"},{\"idx\":1,\"name\":\"hire_year\"}]},\"num_attrs\":2}}",
         "name": "year_union",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        },
        {
         "metadata": "{\"ml_attr\":{\"num_attrs\":2}}",
         "name": "scaled_year",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        },
        {
         "metadata": "{}",
         "name": "scaled_birth",
         "type": "\"double\""
        },
        {
         "metadata": "{}",
         "name": "scaled_hire",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"attrs\":{\"numeric\":[{\"idx\":3,\"name\":\"scaled_birth\"},{\"idx\":4,\"name\":\"scaled_hire\"}],\"nominal\":[{\"vals\":[\"Staff\",\"Senior Engineer\",\"Engineer\",\"Senior Staff\",\"Technique Leader\",\"Assistant Engineer\",\"Manager\"],\"idx\":0,\"name\":\"level_indexed\"},{\"vals\":[\"M\",\"F\"],\"idx\":1,\"name\":\"sex_indexed\"},{\"vals\":[\"Development\",\"Production\",\"Sales\",\"Customer Service\",\"Research\",\"Marketing\",\"Quality Management\",\"Human Resources\",\"Finance\"],\"idx\":2,\"name\":\"dept_indexed\"}]},\"num_attrs\":5}}",
         "name": "predictors",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        }
       ],
       "type": "table"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(emp_df.limit(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "87cae958-d6d4-4835-b1f9-b439e6c47b8f",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Shape after duplicates dropped [331603, 19]\n"
     ]
    },
    {
     "output_type": "display_data",
     "data": {
      "text/html": [
       "<style scoped>\n",
       "  .table-result-container {\n",
       "    max-height: 300px;\n",
       "    overflow: auto;\n",
       "  }\n",
       "  table, th, td {\n",
       "    border: 1px solid black;\n",
       "    border-collapse: collapse;\n",
       "  }\n",
       "  th, td {\n",
       "    padding: 5px;\n",
       "  }\n",
       "  th {\n",
       "    text-align: left;\n",
       "  }\n",
       "</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>emp_no</th><th>emp_level</th><th>birth_date</th><th>first_name</th><th>last_name</th><th>sex</th><th>hire_date</th><th>salary</th><th>department</th><th>level_indexed</th><th>sex_indexed</th><th>dept_indexed</th><th>birth_year</th><th>hire_year</th><th>year_union</th><th>scaled_year</th><th>scaled_birth</th><th>scaled_hire</th><th>predictors</th></tr></thead><tbody><tr><td>10010</td><td>Senior Staff</td><td>1956-04-24</td><td>Kwangyoen</td><td>Speek</td><td>F</td><td>1993-02-14</td><td>72488</td><td>Production</td><td>3.0</td><td>1.0</td><td>1.0</td><td>1956</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1956.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.5384615384615385, 2.6666666666666665))</td><td>1.5384615384615385</td><td>1.5384615384615385</td><td>Map(vectorType -> dense, length -> 5, values -> List(3.0, 1.0, 1.0, 1.5384615384615385, 1.5384615384615385))</td></tr><tr><td>10011</td><td>Engineer</td><td>1961-12-31</td><td>Shuichi</td><td>Tyugu</td><td>F</td><td>1995-01-17</td><td>42365</td><td>Customer Service</td><td>2.0</td><td>1.0</td><td>3.0</td><td>1961</td><td>1995</td><td>Map(vectorType -> dense, length -> 2, values -> List(1961.0, 1995.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(3.4615384615384617, 3.333333333333333))</td><td>3.4615384615384617</td><td>3.4615384615384617</td><td>Map(vectorType -> dense, length -> 5, values -> List(2.0, 1.0, 3.0, 3.4615384615384617, 3.4615384615384617))</td></tr><tr><td>10013</td><td>Senior Engineer</td><td>1961-08-30</td><td>Perry</td><td>Lorho</td><td>F</td><td>1991-08-03</td><td>40000</td><td>Human Resources</td><td>1.0</td><td>1.0</td><td>7.0</td><td>1961</td><td>1991</td><td>Map(vectorType -> dense, length -> 2, values -> List(1961.0, 1991.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(3.4615384615384617, 2.0))</td><td>3.4615384615384617</td><td>3.4615384615384617</td><td>Map(vectorType -> dense, length -> 5, values -> List(1.0, 1.0, 7.0, 3.4615384615384617, 3.4615384615384617))</td></tr><tr><td>10016</td><td>Technique Leader</td><td>1963-07-12</td><td>Bilhanan</td><td>Wuwongse</td><td>M</td><td>1993-10-06</td><td>70889</td><td>Sales</td><td>4.0</td><td>0.0</td><td>2.0</td><td>1963</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1963.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(4.230769230769231, 2.6666666666666665))</td><td>4.230769230769231</td><td>4.230769230769231</td><td>Map(vectorType -> dense, length -> 5, values -> List(4.0, 0.0, 2.0, 4.230769230769231, 4.230769230769231))</td></tr><tr><td>10004</td><td>Staff</td><td>1957-09-28</td><td>Xiong</td><td>Verhoeff</td><td>M</td><td>1987-11-26</td><td>40054</td><td>Production</td><td>0.0</td><td>0.0</td><td>1.0</td><td>1957</td><td>1987</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1987.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 0.6666666666666666))</td><td>1.9230769230769231</td><td>1.9230769230769231</td><td>Map(vectorType -> dense, length -> 5, values -> List(0.0, 0.0, 1.0, 1.9230769230769231, 1.9230769230769231))</td></tr><tr><td>10018</td><td>Engineer</td><td>1957-12-11</td><td>Naftali</td><td>Dulli</td><td>M</td><td>1993-06-06</td><td>55881</td><td>Production</td><td>2.0</td><td>0.0</td><td>1.0</td><td>1957</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 2.6666666666666665))</td><td>1.9230769230769231</td><td>1.9230769230769231</td><td>Map(vectorType -> dense, length -> 5, values -> List(2.0, 0.0, 1.0, 1.9230769230769231, 1.9230769230769231))</td></tr><tr><td>10003</td><td>Engineer</td><td>1958-01-30</td><td>Berry</td><td>Babb</td><td>F</td><td>1992-03-21</td><td>40006</td><td>Production</td><td>2.0</td><td>1.0</td><td>1.0</td><td>1958</td><td>1992</td><td>Map(vectorType -> dense, length -> 2, values -> List(1958.0, 1992.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(2.307692307692308, 2.333333333333333))</td><td>2.307692307692308</td><td>2.307692307692308</td><td>Map(vectorType -> dense, length -> 5, values -> List(2.0, 1.0, 1.0, 2.307692307692308, 2.307692307692308))</td></tr><tr><td>10002</td><td>Engineer</td><td>1954-11-18</td><td>Byong</td><td>Delgrande</td><td>F</td><td>1991-09-07</td><td>65828</td><td>Sales</td><td>2.0</td><td>1.0</td><td>2.0</td><td>1954</td><td>1991</td><td>Map(vectorType -> dense, length -> 2, values -> List(1954.0, 1991.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.7692307692307693, 2.0))</td><td>0.7692307692307693</td><td>0.7692307692307693</td><td>Map(vectorType -> dense, length -> 5, values -> List(2.0, 1.0, 2.0, 0.7692307692307693, 0.7692307692307693))</td></tr><tr><td>10015</td><td>Engineer</td><td>1959-05-19</td><td>Bojan</td><td>Zallocco</td><td>M</td><td>1986-10-14</td><td>40000</td><td>Research</td><td>2.0</td><td>0.0</td><td>4.0</td><td>1959</td><td>1986</td><td>Map(vectorType -> dense, length -> 2, values -> List(1959.0, 1986.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(2.6923076923076925, 0.3333333333333333))</td><td>2.6923076923076925</td><td>2.6923076923076925</td><td>Map(vectorType -> dense, length -> 5, values -> List(2.0, 0.0, 4.0, 2.6923076923076925, 2.6923076923076925))</td></tr><tr><td>10006</td><td>Senior Engineer</td><td>1959-10-30</td><td>Eran</td><td>Cusworth</td><td>M</td><td>1986-11-14</td><td>40000</td><td>Development</td><td>1.0</td><td>0.0</td><td>0.0</td><td>1959</td><td>1986</td><td>Map(vectorType -> dense, length -> 2, values -> List(1959.0, 1986.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(2.6923076923076925, 0.3333333333333333))</td><td>2.6923076923076925</td><td>2.6923076923076925</td><td>Map(vectorType -> dense, length -> 5, values -> List(1.0, 0.0, 0.0, 2.6923076923076925, 2.6923076923076925))</td></tr><tr><td>10010</td><td>Senior Staff</td><td>1956-04-24</td><td>Kwangyoen</td><td>Speek</td><td>F</td><td>1993-02-14</td><td>72488</td><td>Quality Management</td><td>3.0</td><td>1.0</td><td>6.0</td><td>1956</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1956.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.5384615384615385, 2.6666666666666665))</td><td>1.5384615384615385</td><td>1.5384615384615385</td><td>Map(vectorType -> dense, length -> 5, values -> List(3.0, 1.0, 6.0, 1.5384615384615385, 1.5384615384615385))</td></tr><tr><td>10014</td><td>Staff</td><td>1958-03-23</td><td>Subbu</td><td>Bultermann</td><td>F</td><td>1988-03-25</td><td>46168</td><td>Development</td><td>0.0</td><td>1.0</td><td>0.0</td><td>1958</td><td>1988</td><td>Map(vectorType -> dense, length -> 2, values -> List(1958.0, 1988.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(2.307692307692308, 1.0))</td><td>2.307692307692308</td><td>2.307692307692308</td><td>Map(vectorType -> dense, length -> 5, values -> List(0.0, 1.0, 0.0, 2.307692307692308, 2.307692307692308))</td></tr><tr><td>10007</td><td>Staff</td><td>1957-04-14</td><td>Christoph</td><td>Parfitt</td><td>M</td><td>1991-06-28</td><td>56724</td><td>Research</td><td>0.0</td><td>0.0</td><td>4.0</td><td>1957</td><td>1991</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1991.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 2.0))</td><td>1.9230769230769231</td><td>1.9230769230769231</td><td>Map(vectorType -> dense, length -> 5, values -> List(0.0, 0.0, 4.0, 1.9230769230769231, 1.9230769230769231))</td></tr><tr><td>10009</td><td>Engineer</td><td>1962-12-18</td><td>Lihong</td><td>Magliocco</td><td>M</td><td>1993-10-23</td><td>60929</td><td>Quality Management</td><td>2.0</td><td>0.0</td><td>6.0</td><td>1962</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1962.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(3.8461538461538463, 2.6666666666666665))</td><td>3.8461538461538463</td><td>3.8461538461538463</td><td>Map(vectorType -> dense, length -> 5, values -> List(2.0, 0.0, 6.0, 3.8461538461538463, 3.8461538461538463))</td></tr><tr><td>10008</td><td>Senior Staff</td><td>1964-11-17</td><td>Xudong</td><td>Samarati</td><td>M</td><td>1985-11-13</td><td>46671</td><td>Development</td><td>3.0</td><td>0.0</td><td>0.0</td><td>1964</td><td>1985</td><td>Map(vectorType -> dense, length -> 2, values -> List(1964.0, 1985.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(4.615384615384616, 0.0))</td><td>4.615384615384616</td><td>4.615384615384616</td><td>Map(vectorType -> dense, length -> 5, values -> List(3.0, 0.0, 0.0, 4.615384615384616, 4.615384615384616))</td></tr><tr><td>10005</td><td>Senior Engineer</td><td>1952-10-28</td><td>Abdelkader</td><td>Baumann</td><td>F</td><td>1991-01-18</td><td>78228</td><td>Human Resources</td><td>1.0</td><td>1.0</td><td>7.0</td><td>1952</td><td>1991</td><td>Map(vectorType -> dense, length -> 2, values -> List(1952.0, 1991.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.0, 2.0))</td><td>0.0</td><td>0.0</td><td>Map(vectorType -> dense, length -> 5, values -> List(1.0, 1.0, 7.0, 0.0, 0.0))</td></tr><tr><td>10012</td><td>Senior Engineer</td><td>1961-10-25</td><td>Abdulah</td><td>Lunn</td><td>M</td><td>1989-04-08</td><td>40000</td><td>Development</td><td>1.0</td><td>0.0</td><td>0.0</td><td>1961</td><td>1989</td><td>Map(vectorType -> dense, length -> 2, values -> List(1961.0, 1989.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(3.4615384615384617, 1.3333333333333333))</td><td>3.4615384615384617</td><td>3.4615384615384617</td><td>Map(vectorType -> dense, length -> 5, values -> List(1.0, 0.0, 0.0, 3.4615384615384617, 3.4615384615384617))</td></tr><tr><td>10017</td><td>Staff</td><td>1964-01-19</td><td>Venkatesan</td><td>Gilg</td><td>M</td><td>1993-06-28</td><td>71380</td><td>Marketing</td><td>0.0</td><td>0.0</td><td>5.0</td><td>1964</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1964.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(4.615384615384616, 2.6666666666666665))</td><td>4.615384615384616</td><td>4.615384615384616</td><td>Map(vectorType -> dense, length -> 5, values -> List(0.0, 0.0, 5.0, 4.615384615384616, 4.615384615384616))</td></tr><tr><td>10018</td><td>Engineer</td><td>1957-12-11</td><td>Naftali</td><td>Dulli</td><td>M</td><td>1993-06-06</td><td>55881</td><td>Development</td><td>2.0</td><td>0.0</td><td>0.0</td><td>1957</td><td>1993</td><td>Map(vectorType -> dense, length -> 2, values -> List(1957.0, 1993.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(1.9230769230769231, 2.6666666666666665))</td><td>1.9230769230769231</td><td>1.9230769230769231</td><td>Map(vectorType -> dense, length -> 5, values -> List(2.0, 0.0, 0.0, 1.9230769230769231, 1.9230769230769231))</td></tr><tr><td>10001</td><td>Staff</td><td>1953-07-25</td><td>Hideyuki</td><td>Zallocco</td><td>M</td><td>1990-04-28</td><td>60117</td><td>Development</td><td>0.0</td><td>0.0</td><td>0.0</td><td>1953</td><td>1990</td><td>Map(vectorType -> dense, length -> 2, values -> List(1953.0, 1990.0))</td><td>Map(vectorType -> dense, length -> 2, values -> List(0.38461538461538464, 1.6666666666666665))</td><td>0.38461538461538464</td><td>0.38461538461538464</td><td>Map(vectorType -> sparse, length -> 5, indices -> List(3, 4), values -> List(0.38461538461538464, 0.38461538461538464))</td></tr></tbody></table></div>"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "addedWidgets": {},
       "aggData": [],
       "aggError": "",
       "aggOverflow": false,
       "aggSchema": [],
       "aggSeriesLimitReached": false,
       "aggType": "",
       "arguments": {},
       "columnCustomDisplayInfos": {},
       "data": [
        [
         10010,
         "Senior Staff",
         "1956-04-24",
         "Kwangyoen",
         "Speek",
         "F",
         "1993-02-14",
         72488,
         "Production",
         3.0,
         1.0,
         1.0,
         1956,
         1993,
         {
          "length": 2,
          "values": [
           1956.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.5384615384615385,
           2.6666666666666665
          ],
          "vectorType": "dense"
         },
         1.5384615384615385,
         1.5384615384615385,
         {
          "length": 5,
          "values": [
           3.0,
           1.0,
           1.0,
           1.5384615384615385,
           1.5384615384615385
          ],
          "vectorType": "dense"
         }
        ],
        [
         10011,
         "Engineer",
         "1961-12-31",
         "Shuichi",
         "Tyugu",
         "F",
         "1995-01-17",
         42365,
         "Customer Service",
         2.0,
         1.0,
         3.0,
         1961,
         1995,
         {
          "length": 2,
          "values": [
           1961.0,
           1995.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           3.4615384615384617,
           3.333333333333333
          ],
          "vectorType": "dense"
         },
         3.4615384615384617,
         3.4615384615384617,
         {
          "length": 5,
          "values": [
           2.0,
           1.0,
           3.0,
           3.4615384615384617,
           3.4615384615384617
          ],
          "vectorType": "dense"
         }
        ],
        [
         10013,
         "Senior Engineer",
         "1961-08-30",
         "Perry",
         "Lorho",
         "F",
         "1991-08-03",
         40000,
         "Human Resources",
         1.0,
         1.0,
         7.0,
         1961,
         1991,
         {
          "length": 2,
          "values": [
           1961.0,
           1991.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           3.4615384615384617,
           2.0
          ],
          "vectorType": "dense"
         },
         3.4615384615384617,
         3.4615384615384617,
         {
          "length": 5,
          "values": [
           1.0,
           1.0,
           7.0,
           3.4615384615384617,
           3.4615384615384617
          ],
          "vectorType": "dense"
         }
        ],
        [
         10016,
         "Technique Leader",
         "1963-07-12",
         "Bilhanan",
         "Wuwongse",
         "M",
         "1993-10-06",
         70889,
         "Sales",
         4.0,
         0.0,
         2.0,
         1963,
         1993,
         {
          "length": 2,
          "values": [
           1963.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           4.230769230769231,
           2.6666666666666665
          ],
          "vectorType": "dense"
         },
         4.230769230769231,
         4.230769230769231,
         {
          "length": 5,
          "values": [
           4.0,
           0.0,
           2.0,
           4.230769230769231,
           4.230769230769231
          ],
          "vectorType": "dense"
         }
        ],
        [
         10004,
         "Staff",
         "1957-09-28",
         "Xiong",
         "Verhoeff",
         "M",
         "1987-11-26",
         40054,
         "Production",
         0.0,
         0.0,
         1.0,
         1957,
         1987,
         {
          "length": 2,
          "values": [
           1957.0,
           1987.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           0.6666666666666666
          ],
          "vectorType": "dense"
         },
         1.9230769230769231,
         1.9230769230769231,
         {
          "length": 5,
          "values": [
           0.0,
           0.0,
           1.0,
           1.9230769230769231,
           1.9230769230769231
          ],
          "vectorType": "dense"
         }
        ],
        [
         10018,
         "Engineer",
         "1957-12-11",
         "Naftali",
         "Dulli",
         "M",
         "1993-06-06",
         55881,
         "Production",
         2.0,
         0.0,
         1.0,
         1957,
         1993,
         {
          "length": 2,
          "values": [
           1957.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           2.6666666666666665
          ],
          "vectorType": "dense"
         },
         1.9230769230769231,
         1.9230769230769231,
         {
          "length": 5,
          "values": [
           2.0,
           0.0,
           1.0,
           1.9230769230769231,
           1.9230769230769231
          ],
          "vectorType": "dense"
         }
        ],
        [
         10003,
         "Engineer",
         "1958-01-30",
         "Berry",
         "Babb",
         "F",
         "1992-03-21",
         40006,
         "Production",
         2.0,
         1.0,
         1.0,
         1958,
         1992,
         {
          "length": 2,
          "values": [
           1958.0,
           1992.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           2.307692307692308,
           2.333333333333333
          ],
          "vectorType": "dense"
         },
         2.307692307692308,
         2.307692307692308,
         {
          "length": 5,
          "values": [
           2.0,
           1.0,
           1.0,
           2.307692307692308,
           2.307692307692308
          ],
          "vectorType": "dense"
         }
        ],
        [
         10002,
         "Engineer",
         "1954-11-18",
         "Byong",
         "Delgrande",
         "F",
         "1991-09-07",
         65828,
         "Sales",
         2.0,
         1.0,
         2.0,
         1954,
         1991,
         {
          "length": 2,
          "values": [
           1954.0,
           1991.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.7692307692307693,
           2.0
          ],
          "vectorType": "dense"
         },
         0.7692307692307693,
         0.7692307692307693,
         {
          "length": 5,
          "values": [
           2.0,
           1.0,
           2.0,
           0.7692307692307693,
           0.7692307692307693
          ],
          "vectorType": "dense"
         }
        ],
        [
         10015,
         "Engineer",
         "1959-05-19",
         "Bojan",
         "Zallocco",
         "M",
         "1986-10-14",
         40000,
         "Research",
         2.0,
         0.0,
         4.0,
         1959,
         1986,
         {
          "length": 2,
          "values": [
           1959.0,
           1986.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           2.6923076923076925,
           0.3333333333333333
          ],
          "vectorType": "dense"
         },
         2.6923076923076925,
         2.6923076923076925,
         {
          "length": 5,
          "values": [
           2.0,
           0.0,
           4.0,
           2.6923076923076925,
           2.6923076923076925
          ],
          "vectorType": "dense"
         }
        ],
        [
         10006,
         "Senior Engineer",
         "1959-10-30",
         "Eran",
         "Cusworth",
         "M",
         "1986-11-14",
         40000,
         "Development",
         1.0,
         0.0,
         0.0,
         1959,
         1986,
         {
          "length": 2,
          "values": [
           1959.0,
           1986.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           2.6923076923076925,
           0.3333333333333333
          ],
          "vectorType": "dense"
         },
         2.6923076923076925,
         2.6923076923076925,
         {
          "length": 5,
          "values": [
           1.0,
           0.0,
           0.0,
           2.6923076923076925,
           2.6923076923076925
          ],
          "vectorType": "dense"
         }
        ],
        [
         10010,
         "Senior Staff",
         "1956-04-24",
         "Kwangyoen",
         "Speek",
         "F",
         "1993-02-14",
         72488,
         "Quality Management",
         3.0,
         1.0,
         6.0,
         1956,
         1993,
         {
          "length": 2,
          "values": [
           1956.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.5384615384615385,
           2.6666666666666665
          ],
          "vectorType": "dense"
         },
         1.5384615384615385,
         1.5384615384615385,
         {
          "length": 5,
          "values": [
           3.0,
           1.0,
           6.0,
           1.5384615384615385,
           1.5384615384615385
          ],
          "vectorType": "dense"
         }
        ],
        [
         10014,
         "Staff",
         "1958-03-23",
         "Subbu",
         "Bultermann",
         "F",
         "1988-03-25",
         46168,
         "Development",
         0.0,
         1.0,
         0.0,
         1958,
         1988,
         {
          "length": 2,
          "values": [
           1958.0,
           1988.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           2.307692307692308,
           1.0
          ],
          "vectorType": "dense"
         },
         2.307692307692308,
         2.307692307692308,
         {
          "length": 5,
          "values": [
           0.0,
           1.0,
           0.0,
           2.307692307692308,
           2.307692307692308
          ],
          "vectorType": "dense"
         }
        ],
        [
         10007,
         "Staff",
         "1957-04-14",
         "Christoph",
         "Parfitt",
         "M",
         "1991-06-28",
         56724,
         "Research",
         0.0,
         0.0,
         4.0,
         1957,
         1991,
         {
          "length": 2,
          "values": [
           1957.0,
           1991.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           2.0
          ],
          "vectorType": "dense"
         },
         1.9230769230769231,
         1.9230769230769231,
         {
          "length": 5,
          "values": [
           0.0,
           0.0,
           4.0,
           1.9230769230769231,
           1.9230769230769231
          ],
          "vectorType": "dense"
         }
        ],
        [
         10009,
         "Engineer",
         "1962-12-18",
         "Lihong",
         "Magliocco",
         "M",
         "1993-10-23",
         60929,
         "Quality Management",
         2.0,
         0.0,
         6.0,
         1962,
         1993,
         {
          "length": 2,
          "values": [
           1962.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           3.8461538461538463,
           2.6666666666666665
          ],
          "vectorType": "dense"
         },
         3.8461538461538463,
         3.8461538461538463,
         {
          "length": 5,
          "values": [
           2.0,
           0.0,
           6.0,
           3.8461538461538463,
           3.8461538461538463
          ],
          "vectorType": "dense"
         }
        ],
        [
         10008,
         "Senior Staff",
         "1964-11-17",
         "Xudong",
         "Samarati",
         "M",
         "1985-11-13",
         46671,
         "Development",
         3.0,
         0.0,
         0.0,
         1964,
         1985,
         {
          "length": 2,
          "values": [
           1964.0,
           1985.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           4.615384615384616,
           0.0
          ],
          "vectorType": "dense"
         },
         4.615384615384616,
         4.615384615384616,
         {
          "length": 5,
          "values": [
           3.0,
           0.0,
           0.0,
           4.615384615384616,
           4.615384615384616
          ],
          "vectorType": "dense"
         }
        ],
        [
         10005,
         "Senior Engineer",
         "1952-10-28",
         "Abdelkader",
         "Baumann",
         "F",
         "1991-01-18",
         78228,
         "Human Resources",
         1.0,
         1.0,
         7.0,
         1952,
         1991,
         {
          "length": 2,
          "values": [
           1952.0,
           1991.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.0,
           2.0
          ],
          "vectorType": "dense"
         },
         0.0,
         0.0,
         {
          "length": 5,
          "values": [
           1.0,
           1.0,
           7.0,
           0.0,
           0.0
          ],
          "vectorType": "dense"
         }
        ],
        [
         10012,
         "Senior Engineer",
         "1961-10-25",
         "Abdulah",
         "Lunn",
         "M",
         "1989-04-08",
         40000,
         "Development",
         1.0,
         0.0,
         0.0,
         1961,
         1989,
         {
          "length": 2,
          "values": [
           1961.0,
           1989.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           3.4615384615384617,
           1.3333333333333333
          ],
          "vectorType": "dense"
         },
         3.4615384615384617,
         3.4615384615384617,
         {
          "length": 5,
          "values": [
           1.0,
           0.0,
           0.0,
           3.4615384615384617,
           3.4615384615384617
          ],
          "vectorType": "dense"
         }
        ],
        [
         10017,
         "Staff",
         "1964-01-19",
         "Venkatesan",
         "Gilg",
         "M",
         "1993-06-28",
         71380,
         "Marketing",
         0.0,
         0.0,
         5.0,
         1964,
         1993,
         {
          "length": 2,
          "values": [
           1964.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           4.615384615384616,
           2.6666666666666665
          ],
          "vectorType": "dense"
         },
         4.615384615384616,
         4.615384615384616,
         {
          "length": 5,
          "values": [
           0.0,
           0.0,
           5.0,
           4.615384615384616,
           4.615384615384616
          ],
          "vectorType": "dense"
         }
        ],
        [
         10018,
         "Engineer",
         "1957-12-11",
         "Naftali",
         "Dulli",
         "M",
         "1993-06-06",
         55881,
         "Development",
         2.0,
         0.0,
         0.0,
         1957,
         1993,
         {
          "length": 2,
          "values": [
           1957.0,
           1993.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           1.9230769230769231,
           2.6666666666666665
          ],
          "vectorType": "dense"
         },
         1.9230769230769231,
         1.9230769230769231,
         {
          "length": 5,
          "values": [
           2.0,
           0.0,
           0.0,
           1.9230769230769231,
           1.9230769230769231
          ],
          "vectorType": "dense"
         }
        ],
        [
         10001,
         "Staff",
         "1953-07-25",
         "Hideyuki",
         "Zallocco",
         "M",
         "1990-04-28",
         60117,
         "Development",
         0.0,
         0.0,
         0.0,
         1953,
         1990,
         {
          "length": 2,
          "values": [
           1953.0,
           1990.0
          ],
          "vectorType": "dense"
         },
         {
          "length": 2,
          "values": [
           0.38461538461538464,
           1.6666666666666665
          ],
          "vectorType": "dense"
         },
         0.38461538461538464,
         0.38461538461538464,
         {
          "indices": [
           3,
           4
          ],
          "length": 5,
          "values": [
           0.38461538461538464,
           0.38461538461538464
          ],
          "vectorType": "sparse"
         }
        ]
       ],
       "datasetInfos": [],
       "dbfsResultPath": null,
       "isJsonSchema": true,
       "metadata": {},
       "overflow": false,
       "plotOptions": {
        "customPlotOptions": {},
        "displayType": "table",
        "pivotAggregation": null,
        "pivotColumns": null,
        "xColumns": null,
        "yColumns": null
       },
       "removedWidgets": [],
       "schema": [
        {
         "metadata": "{}",
         "name": "emp_no",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "emp_level",
         "type": "\"string\""
        },
        {
         "metadata": "{\"__detected_date_formats\":\"yyyy-M-d\"}",
         "name": "birth_date",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "first_name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "last_name",
         "type": "\"string\""
        },
        {
         "metadata": "{}",
         "name": "sex",
         "type": "\"string\""
        },
        {
         "metadata": "{\"__detected_date_formats\":\"yyyy-M-d\"}",
         "name": "hire_date",
         "type": "\"date\""
        },
        {
         "metadata": "{}",
         "name": "salary",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "department",
         "type": "\"string\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"Staff\",\"Senior Engineer\",\"Engineer\",\"Senior Staff\",\"Technique Leader\",\"Assistant Engineer\",\"Manager\"],\"type\":\"nominal\",\"name\":\"level_indexed\"}}",
         "name": "level_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"M\",\"F\"],\"type\":\"nominal\",\"name\":\"sex_indexed\"}}",
         "name": "sex_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"vals\":[\"Development\",\"Production\",\"Sales\",\"Customer Service\",\"Research\",\"Marketing\",\"Quality Management\",\"Human Resources\",\"Finance\"],\"type\":\"nominal\",\"name\":\"dept_indexed\"}}",
         "name": "dept_indexed",
         "type": "\"double\""
        },
        {
         "metadata": "{}",
         "name": "birth_year",
         "type": "\"integer\""
        },
        {
         "metadata": "{}",
         "name": "hire_year",
         "type": "\"integer\""
        },
        {
         "metadata": "{\"ml_attr\":{\"attrs\":{\"numeric\":[{\"idx\":0,\"name\":\"birth_year\"},{\"idx\":1,\"name\":\"hire_year\"}]},\"num_attrs\":2}}",
         "name": "year_union",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        },
        {
         "metadata": "{\"ml_attr\":{\"num_attrs\":2}}",
         "name": "scaled_year",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        },
        {
         "metadata": "{}",
         "name": "scaled_birth",
         "type": "\"double\""
        },
        {
         "metadata": "{}",
         "name": "scaled_hire",
         "type": "\"double\""
        },
        {
         "metadata": "{\"ml_attr\":{\"attrs\":{\"numeric\":[{\"idx\":3,\"name\":\"scaled_birth\"},{\"idx\":4,\"name\":\"scaled_hire\"}],\"nominal\":[{\"vals\":[\"Staff\",\"Senior Engineer\",\"Engineer\",\"Senior Staff\",\"Technique Leader\",\"Assistant Engineer\",\"Manager\"],\"idx\":0,\"name\":\"level_indexed\"},{\"vals\":[\"M\",\"F\"],\"idx\":1,\"name\":\"sex_indexed\"},{\"vals\":[\"Development\",\"Production\",\"Sales\",\"Customer Service\",\"Research\",\"Marketing\",\"Quality Management\",\"Human Resources\",\"Finance\"],\"idx\":2,\"name\":\"dept_indexed\"}]},\"num_attrs\":5}}",
         "name": "predictors",
         "type": "{\"type\":\"udt\",\"class\":\"org.apache.spark.ml.linalg.VectorUDT\",\"pyClass\":\"pyspark.ml.linalg.VectorUDT\",\"sqlType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"type\",\"type\":\"byte\",\"nullable\":false,\"metadata\":{}},{\"name\":\"size\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"indices\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}},{\"name\":\"values\",\"type\":{\"type\":\"array\",\"elementType\":\"double\",\"containsNull\":false},\"nullable\":true,\"metadata\":{}}]}}"
        }
       ],
       "type": "table"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "emp_df = emp_df.distinct() #remove any duplicates\n",
    "print(\"Shape after duplicates dropped\", shape(emp_df))\n",
    "\n",
    "display(emp_df.limit(20))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "85e92bf8-0627-4964-87bc-4de48f47e196",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "source": [
    "MODEL DEVELOPEMENT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "0df3caa7-5a0c-4aa9-96b6-a0ef70dfcb69",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.regression import LinearRegression, RandomForestRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "680ee349-d7a0-47b1-8a47-d1a51da6608a",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------+-----+\n|hire_year|count|\n+---------+-----+\n|     1986|40004|\n|     1985|39069|\n|     1987|36984|\n|     1988|34791|\n|     1989|31343|\n|     1990|28295|\n|     1991|24921|\n|     1992|22563|\n|     1993|19657|\n|     1994|16303|\n|     1995|13403|\n|     1996|10615|\n|     1997| 7371|\n|     1998| 4611|\n|     1999| 1658|\n|     2000|   15|\n+---------+-----+\n\n"
     ]
    }
   ],
   "source": [
    "emp_df.groupBy(\"hire_year\").count().orderBy('count', ascending=False).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "27502bf1-2972-4809-bcc3-cab7fee3a1f6",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-----+\n|birth_year|count|\n+----------+-----+\n|      1959|25785|\n|      1954|25756|\n|      1958|25706|\n|      1960|25608|\n|      1955|25522|\n|      1961|25487|\n|      1963|25484|\n|      1956|25478|\n|      1962|25398|\n|      1964|25343|\n|      1953|25294|\n|      1957|25215|\n|      1952|23401|\n|      1965| 2126|\n+----------+-----+\n\n"
     ]
    }
   ],
   "source": [
    "emp_df.groupBy(\"birth_year\").count().orderBy('count', ascending=False).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "bdaa2035-3d34-4b6c-958c-542a14cae327",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+\n|          predictors|salary|\n+--------------------+------+\n|[3.0,1.0,1.0,1.53...| 72488|\n|[2.0,1.0,3.0,3.46...| 42365|\n|[1.0,1.0,7.0,3.46...| 40000|\n|[4.0,0.0,2.0,4.23...| 70889|\n|[0.0,0.0,1.0,1.92...| 40054|\n|[2.0,0.0,1.0,1.92...| 55881|\n|[2.0,1.0,1.0,2.30...| 40006|\n|[2.0,1.0,2.0,0.76...| 65828|\n|[2.0,0.0,4.0,2.69...| 40000|\n| (5,[0,2],[4.0,4.0])| 44276|\n|[1.0,0.0,0.0,2.69...| 40000|\n|[3.0,1.0,6.0,1.53...| 72488|\n|[0.0,1.0,0.0,2.30...| 46168|\n|[0.0,0.0,4.0,1.92...| 56724|\n|[2.0,0.0,6.0,3.84...| 60929|\n|[3.0,0.0,0.0,4.61...| 46671|\n|[1.0,1.0,7.0,0.0,...| 78228|\n|[1.0,0.0,0.0,3.46...| 40000|\n|[0.0,0.0,5.0,4.61...| 71380|\n|[2.0,0.0,0.0,1.92...| 55881|\n+--------------------+------+\nonly showing top 20 rows\n\n"
     ]
    }
   ],
   "source": [
    "#To prevent bias and data leakages\n",
    "train_data = emp_df.filter(col(\"hire_year\") < 1996)\n",
    "train_data = train_data.select(\"predictors\", 'salary')\n",
    "\n",
    "train_data.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "a9013975-63d0-4e5d-a439-478c400c2671",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+\n|          predictors|salary|\n+--------------------+------+\n|[2.0,1.0,0.0,0.76...| 44978|\n|[2.0,0.0,0.0,2.69...| 42284|\n|[2.0,0.0,8.0,0.38...| 48041|\n|       (5,[0],[3.0])| 54615|\n|(5,[3,4],[3.46153...| 70303|\n|[1.0,1.0,1.0,3.46...| 40000|\n|[0.0,1.0,6.0,1.53...| 40000|\n|[2.0,1.0,7.0,2.30...| 46836|\n|[1.0,1.0,1.0,3.07...| 47585|\n|[2.0,0.0,6.0,2.69...| 40000|\n|(5,[3,4],[3.84615...| 53977|\n|[2.0,0.0,2.0,3.84...| 66415|\n| (5,[0,2],[3.0,4.0])| 54615|\n|[5.0,0.0,0.0,3.84...| 40000|\n|[3.0,0.0,6.0,3.46...| 40000|\n|(5,[3,4],[3.84615...| 40000|\n|[2.0,0.0,0.0,4.61...| 47280|\n|[2.0,0.0,0.0,2.69...| 40000|\n|[0.0,1.0,1.0,1.53...| 40000|\n|[2.0,1.0,0.0,1.92...| 48857|\n+--------------------+------+\nonly showing top 20 rows\n\n"
     ]
    }
   ],
   "source": [
    "test_data = emp_df.filter(col(\"hire_year\") >= 1996)\n",
    "test_data = test_data.select(\"predictors\", \"salary\")\n",
    "test_data.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "b6a55264-d02b-4c54-b8f3-9b34b89a798b",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "rnd_reg = RandomForestRegressor(featuresCol='predictors', labelCol='salary')\n",
    "rnd_reg = rnd_reg.fit(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "2f432a2b-57c1-459a-905b-172b14579056",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+------------------+\n|          predictors|salary|        prediction|\n+--------------------+------+------------------+\n|[4.0,1.0,2.0,0.0,...| 85292| 67895.47119197769|\n|[1.0,0.0,0.0,0.38...| 40000| 48973.83611162548|\n|[4.0,1.0,4.0,0.76...| 40000|48803.166496493024|\n|[2.0,1.0,0.0,0.76...| 40000| 49059.91280575292|\n|[0.0,1.0,1.0,0.76...| 41716| 48483.99061025144|\n|[0.0,1.0,0.0,0.76...| 41716| 48483.99061025144|\n|[0.0,0.0,2.0,1.15...| 76801| 68426.60490837465|\n|[2.0,0.0,4.0,1.15...| 74934| 48843.58123945023|\n|[1.0,1.0,2.0,1.15...| 80779| 68537.42626531437|\n|[0.0,1.0,2.0,1.53...|108341| 69001.59514485208|\n|[3.0,1.0,1.0,1.53...| 44737| 48704.86431561297|\n|[0.0,0.0,4.0,1.92...| 53048|48731.088173065174|\n|[0.0,1.0,1.0,1.92...| 60157|48612.845065926755|\n|[0.0,1.0,2.0,2.30...| 53882| 68888.21168683951|\n|[2.0,0.0,1.0,2.30...| 44055| 49160.76418723038|\n|[1.0,0.0,0.0,2.30...| 43549| 48896.23947559446|\n|[2.0,1.0,4.0,2.69...| 40164| 49239.56492029775|\n|[0.0,0.0,2.0,2.69...| 46365| 68487.68867236846|\n|[2.0,1.0,2.0,2.69...| 83083| 66638.01199481424|\n|[2.0,1.0,1.0,2.69...| 46722| 49239.56492029775|\n+--------------------+------+------------------+\nonly showing top 20 rows\n\n"
     ]
    }
   ],
   "source": [
    "rnd_preds = rnd_reg.transform(test_data)\n",
    "rnd_preds.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "88f4646c-f5a9-4f0a-9a45-e672ef51a772",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "source": [
    "Train with Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "b06e3e24-15a9-4851-885d-c49f761b2259",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "linear = LinearRegression(featuresCol='predictors', labelCol='salary')\n",
    "linear = linear.fit(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "a35df003-4462-42a0-accf-0c617e1b6547",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+------------------+\n|          predictors|salary|        prediction|\n+--------------------+------+------------------+\n|[4.0,1.0,2.0,0.0,...| 85292| 52753.78109176191|\n|[1.0,0.0,0.0,0.38...| 40000|52137.733613673656|\n|[4.0,1.0,4.0,0.76...| 40000| 53417.16714255408|\n|[2.0,1.0,0.0,0.76...| 40000|  52130.9365282247|\n|[0.0,1.0,1.0,0.76...| 41716| 52478.36470837124|\n|[0.0,1.0,0.0,0.76...| 41716| 52151.63294947606|\n|[0.0,0.0,2.0,1.15...| 76801|  52811.4678750915|\n|[2.0,0.0,4.0,1.15...| 74934|  53444.2349716305|\n|[1.0,1.0,2.0,1.15...| 80779|52799.709523141646|\n|[0.0,1.0,2.0,1.53...|108341| 52815.01900026822|\n|[3.0,1.0,1.0,1.53...| 44737|   52457.242609496|\n|[0.0,0.0,4.0,1.92...| 53048| 53474.85392588366|\n|[0.0,1.0,1.0,1.92...| 60157|52493.248507873934|\n|[0.0,1.0,2.0,2.30...| 53882|52824.941533270015|\n|[2.0,0.0,1.0,2.30...| 44055| 52478.92349444765|\n|[1.0,0.0,0.0,2.30...| 43549| 52162.53994617815|\n|[2.0,1.0,4.0,2.69...| 40164|53462.669896309926|\n|[0.0,0.0,2.0,2.69...| 46365| 52831.31294109509|\n|[2.0,1.0,2.0,2.69...| 83083| 52809.20637851956|\n|[2.0,1.0,1.0,2.69...| 46722| 52482.47461962437|\n+--------------------+------+------------------+\nonly showing top 20 rows\n\n"
     ]
    }
   ],
   "source": [
    "linear_preds = linear.transform(test_data)\n",
    "linear_preds.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "97f22da2-f874-42f6-b233-b5acdcaadbe5",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "source": [
    "METRICS AND EVALUATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "910a1cbe-41db-4e4d-b490-91d49a622f75",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest Mean Squared Error:, 136,852,192\nRandom Forest Mean Absolute Error: 9,412\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "\n",
    "mse_evaluator = RegressionEvaluator(labelCol=\"salary\", predictionCol=\"prediction\", metricName=\"mse\")\n",
    "mae_evaluator = RegressionEvaluator(labelCol=\"salary\", predictionCol=\"prediction\", metricName=\"mae\")\n",
    "\n",
    "rnd_mse = mse_evaluator.evaluate(rnd_preds)\n",
    "rnd_mae = mae_evaluator.evaluate(rnd_preds)\n",
    "\n",
    "print(f\"Random Forest Mean Squared Error:, {int(rnd_mse):,}\")\n",
    "print(\"Random Forest Mean Absolute Error: {:,}\".format(int(rnd_mae)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "8711ca33-4900-47b0-a6d5-2e46bf93001b",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Linear Regression Mean Squared Error:, 206,447,368\nLinear Regression Mean Absolute Error: 11,799\n"
     ]
    }
   ],
   "source": [
    "mse_evaluator = RegressionEvaluator(labelCol=\"salary\", predictionCol=\"prediction\", metricName=\"mse\")\n",
    "mae_evaluator = RegressionEvaluator(labelCol=\"salary\", predictionCol=\"prediction\", metricName=\"mae\")\n",
    "\n",
    "linear_mse = mse_evaluator.evaluate(linear_preds)\n",
    "linear_mae = mae_evaluator.evaluate(linear_preds)\n",
    "\n",
    "print(f\"Linear Regression Mean Squared Error:, {int(linear_mse):,}\")\n",
    "print(\"Linear Regression Mean Absolute Error: {:,}\".format(int(linear_mae)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "7ff34400-db5a-4be8-8434-3ce3810db20a",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "source": [
    "random forest regressor model perfoms better than the linear regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "d4cdd2a0-cf48-40d0-9fa6-45aba0be5abe",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "source": [
    "Testing other pyspark regression models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "de1b6e0c-f465-4094-bd05-b1af4a41b953",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.regression import AFTSurvivalRegression, GBTRegressor, FMRegressor, DecisionTreeRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "408c32ba-cb2f-4abc-828d-dfaed7883189",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m\n",
       "\u001B[0;31mIllegalArgumentException\u001B[0m                  Traceback (most recent call last)\n",
       "File \u001B[0;32m<command-4369324742592617>:2\u001B[0m\n",
       "\u001B[1;32m      1\u001B[0m aft_model \u001B[38;5;241m=\u001B[39m AFTSurvivalRegression(featuresCol\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mpredictors\u001B[39m\u001B[38;5;124m'\u001B[39m, labelCol\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124msalary\u001B[39m\u001B[38;5;124m'\u001B[39m)\n",
       "\u001B[0;32m----> 2\u001B[0m aft_model \u001B[38;5;241m=\u001B[39m aft_model\u001B[38;5;241m.\u001B[39mfit(train_data)\n",
       "\u001B[1;32m      4\u001B[0m aft_preds \u001B[38;5;241m=\u001B[39m aft_model\u001B[38;5;241m.\u001B[39mtransform(test_data)\n",
       "\u001B[1;32m      5\u001B[0m aft_preds\u001B[38;5;241m.\u001B[39mshow()\n",
       "\n",
       "File \u001B[0;32m/databricks/python_shell/dbruntime/MLWorkloadsInstrumentation/_pyspark.py:30\u001B[0m, in \u001B[0;36m_create_patch_function.<locals>.patched_method\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n",
       "\u001B[1;32m     28\u001B[0m call_succeeded \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m\n",
       "\u001B[1;32m     29\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n",
       "\u001B[0;32m---> 30\u001B[0m     result \u001B[38;5;241m=\u001B[39m \u001B[43moriginal_method\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n",
       "\u001B[1;32m     31\u001B[0m     call_succeeded \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n",
       "\u001B[1;32m     32\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m result\n",
       "\n",
       "File \u001B[0;32m/databricks/spark/python/pyspark/ml/base.py:205\u001B[0m, in \u001B[0;36mEstimator.fit\u001B[0;34m(self, dataset, params)\u001B[0m\n",
       "\u001B[1;32m    203\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mcopy(params)\u001B[38;5;241m.\u001B[39m_fit(dataset)\n",
       "\u001B[1;32m    204\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n",
       "\u001B[0;32m--> 205\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_fit\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdataset\u001B[49m\u001B[43m)\u001B[49m\n",
       "\u001B[1;32m    206\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n",
       "\u001B[1;32m    207\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m(\n",
       "\u001B[1;32m    208\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mParams must be either a param map or a list/tuple of param maps, \u001B[39m\u001B[38;5;124m\"\u001B[39m\n",
       "\u001B[1;32m    209\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mbut got \u001B[39m\u001B[38;5;132;01m%s\u001B[39;00m\u001B[38;5;124m.\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m%\u001B[39m \u001B[38;5;28mtype\u001B[39m(params)\n",
       "\u001B[1;32m    210\u001B[0m     )\n",
       "\n",
       "File \u001B[0;32m/databricks/spark/python/pyspark/ml/wrapper.py:383\u001B[0m, in \u001B[0;36mJavaEstimator._fit\u001B[0;34m(self, dataset)\u001B[0m\n",
       "\u001B[1;32m    382\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_fit\u001B[39m(\u001B[38;5;28mself\u001B[39m, dataset: DataFrame) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m JM:\n",
       "\u001B[0;32m--> 383\u001B[0m     java_model \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_fit_java\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdataset\u001B[49m\u001B[43m)\u001B[49m\n",
       "\u001B[1;32m    384\u001B[0m     model \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_create_model(java_model)\n",
       "\u001B[1;32m    385\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_copyValues(model)\n",
       "\n",
       "File \u001B[0;32m/databricks/spark/python/pyspark/ml/wrapper.py:380\u001B[0m, in \u001B[0;36mJavaEstimator._fit_java\u001B[0;34m(self, dataset)\u001B[0m\n",
       "\u001B[1;32m    377\u001B[0m \u001B[38;5;28;01massert\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_java_obj \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n",
       "\u001B[1;32m    379\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_transfer_params_to_java()\n",
       "\u001B[0;32m--> 380\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_java_obj\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfit\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdataset\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_jdf\u001B[49m\u001B[43m)\u001B[49m\n",
       "\n",
       "File \u001B[0;32m/databricks/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/java_gateway.py:1321\u001B[0m, in \u001B[0;36mJavaMember.__call__\u001B[0;34m(self, *args)\u001B[0m\n",
       "\u001B[1;32m   1315\u001B[0m command \u001B[38;5;241m=\u001B[39m proto\u001B[38;5;241m.\u001B[39mCALL_COMMAND_NAME \u001B[38;5;241m+\u001B[39m\\\n",
       "\u001B[1;32m   1316\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mcommand_header \u001B[38;5;241m+\u001B[39m\\\n",
       "\u001B[1;32m   1317\u001B[0m     args_command \u001B[38;5;241m+\u001B[39m\\\n",
       "\u001B[1;32m   1318\u001B[0m     proto\u001B[38;5;241m.\u001B[39mEND_COMMAND_PART\n",
       "\u001B[1;32m   1320\u001B[0m answer \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mgateway_client\u001B[38;5;241m.\u001B[39msend_command(command)\n",
       "\u001B[0;32m-> 1321\u001B[0m return_value \u001B[38;5;241m=\u001B[39m \u001B[43mget_return_value\u001B[49m\u001B[43m(\u001B[49m\n",
       "\u001B[1;32m   1322\u001B[0m \u001B[43m    \u001B[49m\u001B[43manswer\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgateway_client\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtarget_id\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mname\u001B[49m\u001B[43m)\u001B[49m\n",
       "\u001B[1;32m   1324\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m temp_arg \u001B[38;5;129;01min\u001B[39;00m temp_args:\n",
       "\u001B[1;32m   1325\u001B[0m     temp_arg\u001B[38;5;241m.\u001B[39m_detach()\n",
       "\n",
       "File \u001B[0;32m/databricks/spark/python/pyspark/errors/exceptions.py:234\u001B[0m, in \u001B[0;36mcapture_sql_exception.<locals>.deco\u001B[0;34m(*a, **kw)\u001B[0m\n",
       "\u001B[1;32m    230\u001B[0m converted \u001B[38;5;241m=\u001B[39m convert_exception(e\u001B[38;5;241m.\u001B[39mjava_exception)\n",
       "\u001B[1;32m    231\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(converted, UnknownException):\n",
       "\u001B[1;32m    232\u001B[0m     \u001B[38;5;66;03m# Hide where the exception came from that shows a non-Pythonic\u001B[39;00m\n",
       "\u001B[1;32m    233\u001B[0m     \u001B[38;5;66;03m# JVM exception message.\u001B[39;00m\n",
       "\u001B[0;32m--> 234\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m converted \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;28mNone\u001B[39m\n",
       "\u001B[1;32m    235\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n",
       "\u001B[1;32m    236\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m\n",
       "\n",
       "\u001B[0;31mIllegalArgumentException\u001B[0m: censor does not exist. Available: predictors, salary"
      ]
     },
     "metadata": {
      "application/vnd.databricks.v1+output": {
       "arguments": {},
       "data": "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m\n\u001B[0;31mIllegalArgumentException\u001B[0m                  Traceback (most recent call last)\nFile \u001B[0;32m<command-4369324742592617>:2\u001B[0m\n\u001B[1;32m      1\u001B[0m aft_model \u001B[38;5;241m=\u001B[39m AFTSurvivalRegression(featuresCol\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mpredictors\u001B[39m\u001B[38;5;124m'\u001B[39m, labelCol\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124msalary\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[0;32m----> 2\u001B[0m aft_model \u001B[38;5;241m=\u001B[39m aft_model\u001B[38;5;241m.\u001B[39mfit(train_data)\n\u001B[1;32m      4\u001B[0m aft_preds \u001B[38;5;241m=\u001B[39m aft_model\u001B[38;5;241m.\u001B[39mtransform(test_data)\n\u001B[1;32m      5\u001B[0m aft_preds\u001B[38;5;241m.\u001B[39mshow()\n\nFile \u001B[0;32m/databricks/python_shell/dbruntime/MLWorkloadsInstrumentation/_pyspark.py:30\u001B[0m, in \u001B[0;36m_create_patch_function.<locals>.patched_method\u001B[0;34m(self, *args, **kwargs)\u001B[0m\n\u001B[1;32m     28\u001B[0m call_succeeded \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[1;32m     29\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m---> 30\u001B[0m     result \u001B[38;5;241m=\u001B[39m \u001B[43moriginal_method\u001B[49m\u001B[43m(\u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43margs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[38;5;241;43m*\u001B[39;49m\u001B[43mkwargs\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m     31\u001B[0m     call_succeeded \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n\u001B[1;32m     32\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m result\n\nFile \u001B[0;32m/databricks/spark/python/pyspark/ml/base.py:205\u001B[0m, in \u001B[0;36mEstimator.fit\u001B[0;34m(self, dataset, params)\u001B[0m\n\u001B[1;32m    203\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mcopy(params)\u001B[38;5;241m.\u001B[39m_fit(dataset)\n\u001B[1;32m    204\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m--> 205\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_fit\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdataset\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    206\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m    207\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mTypeError\u001B[39;00m(\n\u001B[1;32m    208\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mParams must be either a param map or a list/tuple of param maps, \u001B[39m\u001B[38;5;124m\"\u001B[39m\n\u001B[1;32m    209\u001B[0m         \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mbut got \u001B[39m\u001B[38;5;132;01m%s\u001B[39;00m\u001B[38;5;124m.\u001B[39m\u001B[38;5;124m\"\u001B[39m \u001B[38;5;241m%\u001B[39m \u001B[38;5;28mtype\u001B[39m(params)\n\u001B[1;32m    210\u001B[0m     )\n\nFile \u001B[0;32m/databricks/spark/python/pyspark/ml/wrapper.py:383\u001B[0m, in \u001B[0;36mJavaEstimator._fit\u001B[0;34m(self, dataset)\u001B[0m\n\u001B[1;32m    382\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21m_fit\u001B[39m(\u001B[38;5;28mself\u001B[39m, dataset: DataFrame) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m JM:\n\u001B[0;32m--> 383\u001B[0m     java_model \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_fit_java\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdataset\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m    384\u001B[0m     model \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_create_model(java_model)\n\u001B[1;32m    385\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_copyValues(model)\n\nFile \u001B[0;32m/databricks/spark/python/pyspark/ml/wrapper.py:380\u001B[0m, in \u001B[0;36mJavaEstimator._fit_java\u001B[0;34m(self, dataset)\u001B[0m\n\u001B[1;32m    377\u001B[0m \u001B[38;5;28;01massert\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_java_obj \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m\n\u001B[1;32m    379\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_transfer_params_to_java()\n\u001B[0;32m--> 380\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_java_obj\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mfit\u001B[49m\u001B[43m(\u001B[49m\u001B[43mdataset\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43m_jdf\u001B[49m\u001B[43m)\u001B[49m\n\nFile \u001B[0;32m/databricks/spark/python/lib/py4j-0.10.9.5-src.zip/py4j/java_gateway.py:1321\u001B[0m, in \u001B[0;36mJavaMember.__call__\u001B[0;34m(self, *args)\u001B[0m\n\u001B[1;32m   1315\u001B[0m command \u001B[38;5;241m=\u001B[39m proto\u001B[38;5;241m.\u001B[39mCALL_COMMAND_NAME \u001B[38;5;241m+\u001B[39m\\\n\u001B[1;32m   1316\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mcommand_header \u001B[38;5;241m+\u001B[39m\\\n\u001B[1;32m   1317\u001B[0m     args_command \u001B[38;5;241m+\u001B[39m\\\n\u001B[1;32m   1318\u001B[0m     proto\u001B[38;5;241m.\u001B[39mEND_COMMAND_PART\n\u001B[1;32m   1320\u001B[0m answer \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mgateway_client\u001B[38;5;241m.\u001B[39msend_command(command)\n\u001B[0;32m-> 1321\u001B[0m return_value \u001B[38;5;241m=\u001B[39m \u001B[43mget_return_value\u001B[49m\u001B[43m(\u001B[49m\n\u001B[1;32m   1322\u001B[0m \u001B[43m    \u001B[49m\u001B[43manswer\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mgateway_client\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mtarget_id\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[38;5;28;43mself\u001B[39;49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mname\u001B[49m\u001B[43m)\u001B[49m\n\u001B[1;32m   1324\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m temp_arg \u001B[38;5;129;01min\u001B[39;00m temp_args:\n\u001B[1;32m   1325\u001B[0m     temp_arg\u001B[38;5;241m.\u001B[39m_detach()\n\nFile \u001B[0;32m/databricks/spark/python/pyspark/errors/exceptions.py:234\u001B[0m, in \u001B[0;36mcapture_sql_exception.<locals>.deco\u001B[0;34m(*a, **kw)\u001B[0m\n\u001B[1;32m    230\u001B[0m converted \u001B[38;5;241m=\u001B[39m convert_exception(e\u001B[38;5;241m.\u001B[39mjava_exception)\n\u001B[1;32m    231\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28misinstance\u001B[39m(converted, UnknownException):\n\u001B[1;32m    232\u001B[0m     \u001B[38;5;66;03m# Hide where the exception came from that shows a non-Pythonic\u001B[39;00m\n\u001B[1;32m    233\u001B[0m     \u001B[38;5;66;03m# JVM exception message.\u001B[39;00m\n\u001B[0;32m--> 234\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m converted \u001B[38;5;28;01mfrom\u001B[39;00m \u001B[38;5;28mNone\u001B[39m\n\u001B[1;32m    235\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[1;32m    236\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m\n\n\u001B[0;31mIllegalArgumentException\u001B[0m: censor does not exist. Available: predictors, salary",
       "errorSummary": "<span class='ansi-red-fg'>IllegalArgumentException</span>: censor does not exist. Available: predictors, salary",
       "errorTraceType": "ansi",
       "metadata": {},
       "type": "ipynbError"
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "aft_model = AFTSurvivalRegression(featuresCol='predictors', labelCol='salary')\n",
    "aft_model = aft_model.fit(train_data)\n",
    "\n",
    "aft_preds = aft_model.transform(test_data)\n",
    "aft_preds.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "8dcd3808-b607-40ff-8a1c-54b4821f0388",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+------------------+\n|          predictors|salary|        prediction|\n+--------------------+------+------------------+\n|[4.0,1.0,2.0,0.0,...| 85292| 70473.75006753167|\n|[1.0,0.0,0.0,0.38...| 40000|48648.794502646044|\n|[4.0,1.0,4.0,0.76...| 40000|48437.098027414504|\n|[2.0,1.0,0.0,0.76...| 40000|49008.263003382745|\n|[0.0,1.0,1.0,0.76...| 41716|48685.481707777086|\n|[0.0,1.0,0.0,0.76...| 41716| 48524.33836808741|\n|[0.0,0.0,2.0,1.15...| 76801| 69768.68021143017|\n|[2.0,0.0,4.0,1.15...| 74934| 48953.55134759175|\n|[1.0,1.0,2.0,1.15...| 80779| 69711.43109851463|\n|[0.0,1.0,2.0,1.53...|108341| 69996.44574838014|\n|[3.0,1.0,1.0,1.53...| 44737| 48926.93556549842|\n|[0.0,0.0,4.0,1.92...| 53048|48694.737439996345|\n|[0.0,1.0,1.0,1.92...| 60157|48697.206326061234|\n|[0.0,1.0,2.0,2.30...| 53882| 69886.12822589024|\n|[2.0,0.0,1.0,2.30...| 44055| 48310.45747837725|\n|[1.0,0.0,0.0,2.30...| 43549|  48772.2276804477|\n|[2.0,1.0,4.0,2.69...| 40164| 49090.34988050985|\n|[0.0,0.0,2.0,2.69...| 46365| 69921.61844889313|\n|[2.0,1.0,2.0,2.69...| 83083| 69883.96517032947|\n|[2.0,1.0,1.0,2.69...| 46722| 48970.02694929789|\n+--------------------+------+------------------+\nonly showing top 20 rows\n\n"
     ]
    }
   ],
   "source": [
    "gb_model = GBTRegressor(featuresCol='predictors', labelCol='salary')\n",
    "gb_model = gb_model.fit(train_data)\n",
    "\n",
    "gb_preds = gb_model.transform(test_data)\n",
    "gb_preds.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "fffebb29-ba00-414b-9baf-4114273886cc",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+------------------+\n|          predictors|salary|        prediction|\n+--------------------+------+------------------+\n|[4.0,1.0,2.0,0.0,...| 85292|  9639.47446786482|\n|[1.0,0.0,0.0,0.38...| 40000|1151.5367331463622|\n|[4.0,1.0,4.0,0.76...| 40000|27098.862189138526|\n|[2.0,1.0,0.0,0.76...| 40000| 9724.479254843582|\n|[0.0,1.0,1.0,0.76...| 41716| 5996.174336819629|\n|[0.0,1.0,0.0,0.76...| 41716| 4462.357944149711|\n|[0.0,0.0,2.0,1.15...| 76801| 5135.482425173914|\n|[2.0,0.0,4.0,1.15...| 74934|14851.179371093123|\n|[1.0,1.0,2.0,1.15...| 80779|15363.958735207645|\n|[0.0,1.0,2.0,1.53...|108341|15610.992676347785|\n|[3.0,1.0,1.0,1.53...| 44737| 26471.21785448063|\n|[0.0,0.0,4.0,1.92...| 53048|15185.344072978198|\n|[0.0,1.0,1.0,1.92...| 60157|17788.450634967547|\n|[0.0,1.0,2.0,2.30...| 53882|25907.926698481268|\n|[2.0,0.0,1.0,2.30...| 44055|21850.978354083138|\n|[1.0,0.0,0.0,2.30...| 43549| 14503.23309816224|\n|[2.0,1.0,4.0,2.69...| 40164| 53995.69132552551|\n|[0.0,0.0,2.0,2.69...| 46365| 19465.51384546813|\n|[2.0,1.0,2.0,2.69...| 83083|45566.472359063155|\n|[2.0,1.0,1.0,2.69...| 46722|41351.862875831976|\n+--------------------+------+------------------+\nonly showing top 20 rows\n\n"
     ]
    }
   ],
   "source": [
    "fm_model = FMRegressor(featuresCol='predictors', labelCol='salary')\n",
    "fm_model = fm_model.fit(train_data)\n",
    "\n",
    "fm_preds = fm_model.transform(test_data)\n",
    "fm_preds.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "40a713ab-4c2c-4313-933d-e50a70e9ca5a",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+------------------+\n|          predictors|salary|        prediction|\n+--------------------+------+------------------+\n|[4.0,1.0,2.0,0.0,...| 85292| 70639.92797783934|\n|[1.0,0.0,0.0,0.38...| 40000|48777.889420034204|\n|[4.0,1.0,4.0,0.76...| 40000| 48671.15149802891|\n|[2.0,1.0,0.0,0.76...| 40000|48777.889420034204|\n|[0.0,1.0,1.0,0.76...| 41716| 48671.15149802891|\n|[0.0,1.0,0.0,0.76...| 41716| 48671.15149802891|\n|[0.0,0.0,2.0,1.15...| 76801| 69920.29964252486|\n|[2.0,0.0,4.0,1.15...| 74934| 48986.27496483825|\n|[1.0,1.0,2.0,1.15...| 80779| 69936.88119993183|\n|[0.0,1.0,2.0,1.53...|108341| 69920.29964252486|\n|[3.0,1.0,1.0,1.53...| 44737| 48671.15149802891|\n|[0.0,0.0,4.0,1.92...| 53048| 48671.15149802891|\n|[0.0,1.0,1.0,1.92...| 60157| 48671.15149802891|\n|[0.0,1.0,2.0,2.30...| 53882| 69920.29964252486|\n|[2.0,0.0,1.0,2.30...| 44055|48777.889420034204|\n|[1.0,0.0,0.0,2.30...| 43549|48777.889420034204|\n|[2.0,1.0,4.0,2.69...| 40164| 48986.27496483825|\n|[0.0,0.0,2.0,2.69...| 46365| 69920.29964252486|\n|[2.0,1.0,2.0,2.69...| 83083| 69920.29964252486|\n|[2.0,1.0,1.0,2.69...| 46722|48777.889420034204|\n+--------------------+------+------------------+\nonly showing top 20 rows\n\n"
     ]
    }
   ],
   "source": [
    "tree_model = DecisionTreeRegressor(featuresCol='predictors', labelCol='salary')\n",
    "tree_model = tree_model.fit(train_data)\n",
    "\n",
    "tree_preds = tree_model.transform(test_data)\n",
    "tree_preds.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "21b051e7-fb67-4a1c-8691-240f290e845f",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "source": [
    "Evaluation and Metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "5b0003b7-0f76-4e58-9f8b-564720e2b39d",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosted Tree Mean Squared Error:, 136,010,392\nGradient Boosted Tree Mean Absolute Error: 9,322\n"
     ]
    }
   ],
   "source": [
    "gb_mse = mse_evaluator.evaluate(gb_preds)\n",
    "gb_mae = mae_evaluator.evaluate(gb_preds)\n",
    "\n",
    "print(f\"Gradient Boosted Tree Mean Squared Error:, {int(gb_mse):,}\")\n",
    "print(\"Gradient Boosted Tree Mean Absolute Error: {:,}\".format(int(gb_mae)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "465a1f37-be98-4bb6-bab1-2999224ad03d",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Factorization Algorithm Mean Squared Error:, 1,324,512,142\nFactorization Algorithm Mean Absolute Error: 30,712\n"
     ]
    }
   ],
   "source": [
    "fm_mse = mse_evaluator.evaluate(fm_preds)\n",
    "fm_mae = mae_evaluator.evaluate(fm_preds)\n",
    "\n",
    "print(f\"Factorization Algorithm Mean Squared Error:, {int(fm_mse):,}\")\n",
    "print(\"Factorization Algorithm Mean Absolute Error: {:,}\".format(int(fm_mae)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "e2d95814-c0e1-4ed7-b90c-5e87f6c80a81",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree Mean Squared Error:, 135,881,360\nDecision Tree Mean Absolute Error: 9,318\n"
     ]
    }
   ],
   "source": [
    "tree_mse = mse_evaluator.evaluate(tree_preds)\n",
    "tree_mae = mae_evaluator.evaluate(tree_preds)\n",
    "\n",
    "print(f\"Decision Tree Mean Squared Error:, {int(tree_mse):,}\")\n",
    "print(\"Decision Tree Mean Absolute Error: {:,}\".format(int(tree_mae)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "e6f0d0fd-c601-403c-b1a8-f11788cbbd7e",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "source": [
    "Above all and all, Decision Tree Model produced predictions that are more accurate than the other models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "9c5f9030-8a54-4882-aa5e-9b63e963fa09",
     "showTitle": false,
     "tableResultSettingsMap": {},
     "title": ""
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "environmentMetadata": null,
   "language": "python",
   "notebookMetadata": {
    "pythonIndentUnit": 4
   },
   "notebookName": "basic_ml_on_pyspark",
   "widgets": {}
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}