no-lustre.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scale Features and Build Model\n",
    "### Scales Raw Features\n",
    "\n",
    "Import CSV of Aggregated Darshan Logs <br>\n",
    "Apply Log10 and Percent Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./raws.csv\")\n",
    "#df.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(df.columns[0],axis = 1)\n",
    "df = df.drop(df.columns[0],axis = 1)\n",
    "f = pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>posix_read_time</th>\n",
       "      <th>posix_write_time</th>\n",
       "      <th>posix_meta_time</th>\n",
       "      <th>posix_bytes_read</th>\n",
       "      <th>posix_bytes_read_100</th>\n",
       "      <th>posix_bytes_read_1K</th>\n",
       "      <th>posix_bytes_read_10K</th>\n",
       "      <th>posix_bytes_read_100K</th>\n",
       "      <th>posix_bytes_read_1lM</th>\n",
       "      <th>posix_bytes_read_4M</th>\n",
       "      <th>...</th>\n",
       "      <th>posix_m_align</th>\n",
       "      <th>lustre_number_of_files</th>\n",
       "      <th>lustre_mdts</th>\n",
       "      <th>lustre_osts</th>\n",
       "      <th>lustre_stripe_size</th>\n",
       "      <th>lustre_stripe_offset</th>\n",
       "      <th>lustre_stripe_width</th>\n",
       "      <th>lustre_number_of_osts</th>\n",
       "      <th>jobid</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9042391.0</td>\n",
       "      <td>../results/USER375904U_sss2_id9042391_9-26-414...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>104.611641</td>\n",
       "      <td>10.024055</td>\n",
       "      <td>20.060841</td>\n",
       "      <td>2.390891e+10</td>\n",
       "      <td>147688.0</td>\n",
       "      <td>141853.0</td>\n",
       "      <td>33215.0</td>\n",
       "      <td>261267.0</td>\n",
       "      <td>51049.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>13704.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>147.0</td>\n",
       "      <td>9868242.0</td>\n",
       "      <td>../results/USER382862U_SpEC_id9868242_4-29-741...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>124.560730</td>\n",
       "      <td>42.051125</td>\n",
       "      <td>54.839272</td>\n",
       "      <td>5.019637e+10</td>\n",
       "      <td>332059.0</td>\n",
       "      <td>684893.0</td>\n",
       "      <td>130143.0</td>\n",
       "      <td>919688.0</td>\n",
       "      <td>2515.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>6704.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>186.0</td>\n",
       "      <td>9029683.0</td>\n",
       "      <td>../results/USER264205U_SpEC_id9029683_9-20-580...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>25763.292969</td>\n",
       "      <td>582.297363</td>\n",
       "      <td>24.895737</td>\n",
       "      <td>5.488943e+12</td>\n",
       "      <td>30785.0</td>\n",
       "      <td>767.0</td>\n",
       "      <td>508381.0</td>\n",
       "      <td>2040.0</td>\n",
       "      <td>13683.0</td>\n",
       "      <td>626216.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2320.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>54.993103</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>9165166.0</td>\n",
       "      <td>../results/USER341996U_CCTM_v52_Linux2_x86_64i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>154.534821</td>\n",
       "      <td>681.548279</td>\n",
       "      <td>658.484985</td>\n",
       "      <td>2.293203e+10</td>\n",
       "      <td>588029.0</td>\n",
       "      <td>2926599.0</td>\n",
       "      <td>425127.0</td>\n",
       "      <td>336133.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>6328.0</td>\n",
       "      <td>201.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>201.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>8269019.0</td>\n",
       "      <td>../results/USER141481U_SpEC_id8269019_3-3-4542...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875282</th>\n",
       "      <td>138.354477</td>\n",
       "      <td>82.278084</td>\n",
       "      <td>194.485565</td>\n",
       "      <td>5.593977e+10</td>\n",
       "      <td>216146.0</td>\n",
       "      <td>535152.0</td>\n",
       "      <td>99814.0</td>\n",
       "      <td>889010.0</td>\n",
       "      <td>12485.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9920.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>7759072.0</td>\n",
       "      <td>../results/USER336298U_SpEC_id7759072_10-30-62...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875283</th>\n",
       "      <td>54.443073</td>\n",
       "      <td>231.440857</td>\n",
       "      <td>25.271391</td>\n",
       "      <td>1.465277e+09</td>\n",
       "      <td>3099.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>180662.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8704.0</td>\n",
       "      <td>1088.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1088.0</td>\n",
       "      <td>359.0</td>\n",
       "      <td>7275819.0</td>\n",
       "      <td>../results/USER331454U_pw.x_id7275819_8-2-4790...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875284</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8305062.0</td>\n",
       "      <td>../results/USER371284U_sss2_id8305062_3-14-211...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875285</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>31428.0</td>\n",
       "      <td>../results/USER303986U_testnodes_opt.x_id31428...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875286</th>\n",
       "      <td>227.063828</td>\n",
       "      <td>191.747269</td>\n",
       "      <td>172.671997</td>\n",
       "      <td>1.077412e+11</td>\n",
       "      <td>775359.0</td>\n",
       "      <td>1545229.0</td>\n",
       "      <td>305757.0</td>\n",
       "      <td>1897233.0</td>\n",
       "      <td>1068.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>15104.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>7525550.0</td>\n",
       "      <td>../results/USER382862U_SpEC_id7525550_9-24-380...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>875287 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        posix_read_time  posix_write_time  posix_meta_time  posix_bytes_read  \\\n",
       "0              0.000000          0.000000         0.000000      0.000000e+00   \n",
       "1            104.611641         10.024055        20.060841      2.390891e+10   \n",
       "2            124.560730         42.051125        54.839272      5.019637e+10   \n",
       "3          25763.292969        582.297363        24.895737      5.488943e+12   \n",
       "4            154.534821        681.548279       658.484985      2.293203e+10   \n",
       "...                 ...               ...              ...               ...   \n",
       "875282       138.354477         82.278084       194.485565      5.593977e+10   \n",
       "875283        54.443073        231.440857        25.271391      1.465277e+09   \n",
       "875284         0.000000          0.000000         0.000000      0.000000e+00   \n",
       "875285         0.000000          0.000000         0.000000      0.000000e+00   \n",
       "875286       227.063828        191.747269       172.671997      1.077412e+11   \n",
       "\n",
       "        posix_bytes_read_100  posix_bytes_read_1K  posix_bytes_read_10K  \\\n",
       "0                        0.0                  0.0                   0.0   \n",
       "1                   147688.0             141853.0               33215.0   \n",
       "2                   332059.0             684893.0              130143.0   \n",
       "3                    30785.0                767.0              508381.0   \n",
       "4                   588029.0            2926599.0              425127.0   \n",
       "...                      ...                  ...                   ...   \n",
       "875282              216146.0             535152.0               99814.0   \n",
       "875283                3099.0                  0.0              180662.0   \n",
       "875284                   0.0                  0.0                   0.0   \n",
       "875285                   0.0                  0.0                   0.0   \n",
       "875286              775359.0            1545229.0              305757.0   \n",
       "\n",
       "        posix_bytes_read_100K  posix_bytes_read_1lM  posix_bytes_read_4M  ...  \\\n",
       "0                         0.0                   0.0                  0.0  ...   \n",
       "1                    261267.0               51049.0                  0.0  ...   \n",
       "2                    919688.0                2515.0                  0.0  ...   \n",
       "3                      2040.0               13683.0             626216.0  ...   \n",
       "4                    336133.0                  98.0                  0.0  ...   \n",
       "...                       ...                   ...                  ...  ...   \n",
       "875282               889010.0               12485.0                  0.0  ...   \n",
       "875283                    0.0                   0.0                  0.0  ...   \n",
       "875284                    0.0                   0.0                  0.0  ...   \n",
       "875285                    0.0                   0.0                  0.0  ...   \n",
       "875286              1897233.0                1068.0                  0.0  ...   \n",
       "\n",
       "        posix_m_align  lustre_number_of_files  lustre_mdts  lustre_osts  \\\n",
       "0                 0.0                     4.0          1.0    36.000000   \n",
       "1             13704.0                   176.0          1.0   360.000000   \n",
       "2              6704.0                   224.0          1.0   360.000000   \n",
       "3              2320.0                   290.0          1.0    54.993103   \n",
       "4              6328.0                   201.0          1.0   360.000000   \n",
       "...               ...                     ...          ...          ...   \n",
       "875282         9920.0                   124.0          1.0   360.000000   \n",
       "875283         8704.0                  1088.0          1.0   360.000000   \n",
       "875284            0.0                     5.0          1.0    36.000000   \n",
       "875285            8.0                     2.0          1.0    36.000000   \n",
       "875286        15104.0                   128.0          1.0   360.000000   \n",
       "\n",
       "        lustre_stripe_size  lustre_stripe_offset  lustre_stripe_width  \\\n",
       "0                1048576.0                   0.0                  4.0   \n",
       "1                1048576.0                   0.0                176.0   \n",
       "2                1048576.0                   0.0                224.0   \n",
       "3                1048576.0                   0.0                290.0   \n",
       "4                1048576.0                   0.0                201.0   \n",
       "...                    ...                   ...                  ...   \n",
       "875282           1048576.0                   0.0                124.0   \n",
       "875283           1048576.0                   0.0               1088.0   \n",
       "875284           1048576.0                   0.0                  5.0   \n",
       "875285           1048576.0                   0.0                  2.0   \n",
       "875286           1048576.0                   0.0                128.0   \n",
       "\n",
       "        lustre_number_of_osts      jobid  \\\n",
       "0                         4.0  9042391.0   \n",
       "1                       147.0  9868242.0   \n",
       "2                       186.0  9029683.0   \n",
       "3                        52.0  9165166.0   \n",
       "4                       158.0  8269019.0   \n",
       "...                       ...        ...   \n",
       "875282                  110.0  7759072.0   \n",
       "875283                  359.0  7275819.0   \n",
       "875284                    5.0  8305062.0   \n",
       "875285                    1.0    31428.0   \n",
       "875286                  118.0  7525550.0   \n",
       "\n",
       "                                                     path  \n",
       "0       ../results/USER375904U_sss2_id9042391_9-26-414...  \n",
       "1       ../results/USER382862U_SpEC_id9868242_4-29-741...  \n",
       "2       ../results/USER264205U_SpEC_id9029683_9-20-580...  \n",
       "3       ../results/USER341996U_CCTM_v52_Linux2_x86_64i...  \n",
       "4       ../results/USER141481U_SpEC_id8269019_3-3-4542...  \n",
       "...                                                   ...  \n",
       "875282  ../results/USER336298U_SpEC_id7759072_10-30-62...  \n",
       "875283  ../results/USER331454U_pw.x_id7275819_8-2-4790...  \n",
       "875284  ../results/USER371284U_sss2_id8305062_3-14-211...  \n",
       "875285  ../results/USER303986U_testnodes_opt.x_id31428...  \n",
       "875286  ../results/USER382862U_SpEC_id7525550_9-24-380...  \n",
       "\n",
       "[875287 rows x 50 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>posix_read_time</th>\n",
       "      <th>posix_write_time</th>\n",
       "      <th>posix_meta_time</th>\n",
       "      <th>posix_bytes_read</th>\n",
       "      <th>posix_bytes_read_100</th>\n",
       "      <th>posix_bytes_read_1K</th>\n",
       "      <th>posix_bytes_read_10K</th>\n",
       "      <th>posix_bytes_read_100K</th>\n",
       "      <th>posix_bytes_read_1lM</th>\n",
       "      <th>posix_bytes_read_4M</th>\n",
       "      <th>...</th>\n",
       "      <th>posix_m_align</th>\n",
       "      <th>lustre_number_of_files</th>\n",
       "      <th>lustre_mdts</th>\n",
       "      <th>lustre_osts</th>\n",
       "      <th>lustre_stripe_size</th>\n",
       "      <th>lustre_stripe_offset</th>\n",
       "      <th>lustre_stripe_width</th>\n",
       "      <th>lustre_number_of_osts</th>\n",
       "      <th>jobid</th>\n",
       "      <th>path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9042391.0</td>\n",
       "      <td>../results/USER375904U_sss2_id9042391_9-26-414...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>104.611641</td>\n",
       "      <td>10.024055</td>\n",
       "      <td>20.060841</td>\n",
       "      <td>2.390891e+10</td>\n",
       "      <td>147688.0</td>\n",
       "      <td>141853.0</td>\n",
       "      <td>33215.0</td>\n",
       "      <td>261267.0</td>\n",
       "      <td>51049.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>13704.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>147.0</td>\n",
       "      <td>9868242.0</td>\n",
       "      <td>../results/USER382862U_SpEC_id9868242_4-29-741...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>124.560730</td>\n",
       "      <td>42.051125</td>\n",
       "      <td>54.839272</td>\n",
       "      <td>5.019637e+10</td>\n",
       "      <td>332059.0</td>\n",
       "      <td>684893.0</td>\n",
       "      <td>130143.0</td>\n",
       "      <td>919688.0</td>\n",
       "      <td>2515.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>6704.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>186.0</td>\n",
       "      <td>9029683.0</td>\n",
       "      <td>../results/USER264205U_SpEC_id9029683_9-20-580...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>25763.292969</td>\n",
       "      <td>582.297363</td>\n",
       "      <td>24.895737</td>\n",
       "      <td>5.488943e+12</td>\n",
       "      <td>30785.0</td>\n",
       "      <td>767.0</td>\n",
       "      <td>508381.0</td>\n",
       "      <td>2040.0</td>\n",
       "      <td>13683.0</td>\n",
       "      <td>626216.0</td>\n",
       "      <td>...</td>\n",
       "      <td>2320.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>54.993103</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>9165166.0</td>\n",
       "      <td>../results/USER341996U_CCTM_v52_Linux2_x86_64i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>154.534821</td>\n",
       "      <td>681.548279</td>\n",
       "      <td>658.484985</td>\n",
       "      <td>2.293203e+10</td>\n",
       "      <td>588029.0</td>\n",
       "      <td>2926599.0</td>\n",
       "      <td>425127.0</td>\n",
       "      <td>336133.0</td>\n",
       "      <td>98.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>6328.0</td>\n",
       "      <td>201.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>201.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>8269019.0</td>\n",
       "      <td>../results/USER141481U_SpEC_id8269019_3-3-4542...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875282</th>\n",
       "      <td>138.354477</td>\n",
       "      <td>82.278084</td>\n",
       "      <td>194.485565</td>\n",
       "      <td>5.593977e+10</td>\n",
       "      <td>216146.0</td>\n",
       "      <td>535152.0</td>\n",
       "      <td>99814.0</td>\n",
       "      <td>889010.0</td>\n",
       "      <td>12485.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>9920.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>7759072.0</td>\n",
       "      <td>../results/USER336298U_SpEC_id7759072_10-30-62...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875283</th>\n",
       "      <td>54.443073</td>\n",
       "      <td>231.440857</td>\n",
       "      <td>25.271391</td>\n",
       "      <td>1.465277e+09</td>\n",
       "      <td>3099.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>180662.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8704.0</td>\n",
       "      <td>1088.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1088.0</td>\n",
       "      <td>359.0</td>\n",
       "      <td>7275819.0</td>\n",
       "      <td>../results/USER331454U_pw.x_id7275819_8-2-4790...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875284</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8305062.0</td>\n",
       "      <td>../results/USER371284U_sss2_id8305062_3-14-211...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875285</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>36.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>31428.0</td>\n",
       "      <td>../results/USER303986U_testnodes_opt.x_id31428...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875286</th>\n",
       "      <td>227.063828</td>\n",
       "      <td>191.747269</td>\n",
       "      <td>172.671997</td>\n",
       "      <td>1.077412e+11</td>\n",
       "      <td>775359.0</td>\n",
       "      <td>1545229.0</td>\n",
       "      <td>305757.0</td>\n",
       "      <td>1897233.0</td>\n",
       "      <td>1068.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>15104.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>360.000000</td>\n",
       "      <td>1048576.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>118.0</td>\n",
       "      <td>7525550.0</td>\n",
       "      <td>../results/USER382862U_SpEC_id7525550_9-24-380...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>875287 rows × 50 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        posix_read_time  posix_write_time  posix_meta_time  posix_bytes_read  \\\n",
       "0              0.000000          0.000000         0.000000      0.000000e+00   \n",
       "1            104.611641         10.024055        20.060841      2.390891e+10   \n",
       "2            124.560730         42.051125        54.839272      5.019637e+10   \n",
       "3          25763.292969        582.297363        24.895737      5.488943e+12   \n",
       "4            154.534821        681.548279       658.484985      2.293203e+10   \n",
       "...                 ...               ...              ...               ...   \n",
       "875282       138.354477         82.278084       194.485565      5.593977e+10   \n",
       "875283        54.443073        231.440857        25.271391      1.465277e+09   \n",
       "875284         0.000000          0.000000         0.000000      0.000000e+00   \n",
       "875285         0.000000          0.000000         0.000000      0.000000e+00   \n",
       "875286       227.063828        191.747269       172.671997      1.077412e+11   \n",
       "\n",
       "        posix_bytes_read_100  posix_bytes_read_1K  posix_bytes_read_10K  \\\n",
       "0                        0.0                  0.0                   0.0   \n",
       "1                   147688.0             141853.0               33215.0   \n",
       "2                   332059.0             684893.0              130143.0   \n",
       "3                    30785.0                767.0              508381.0   \n",
       "4                   588029.0            2926599.0              425127.0   \n",
       "...                      ...                  ...                   ...   \n",
       "875282              216146.0             535152.0               99814.0   \n",
       "875283                3099.0                  0.0              180662.0   \n",
       "875284                   0.0                  0.0                   0.0   \n",
       "875285                   0.0                  0.0                   0.0   \n",
       "875286              775359.0            1545229.0              305757.0   \n",
       "\n",
       "        posix_bytes_read_100K  posix_bytes_read_1lM  posix_bytes_read_4M  ...  \\\n",
       "0                         0.0                   0.0                  0.0  ...   \n",
       "1                    261267.0               51049.0                  0.0  ...   \n",
       "2                    919688.0                2515.0                  0.0  ...   \n",
       "3                      2040.0               13683.0             626216.0  ...   \n",
       "4                    336133.0                  98.0                  0.0  ...   \n",
       "...                       ...                   ...                  ...  ...   \n",
       "875282               889010.0               12485.0                  0.0  ...   \n",
       "875283                    0.0                   0.0                  0.0  ...   \n",
       "875284                    0.0                   0.0                  0.0  ...   \n",
       "875285                    0.0                   0.0                  0.0  ...   \n",
       "875286              1897233.0                1068.0                  0.0  ...   \n",
       "\n",
       "        posix_m_align  lustre_number_of_files  lustre_mdts  lustre_osts  \\\n",
       "0                 0.0                     4.0          1.0    36.000000   \n",
       "1             13704.0                   176.0          1.0   360.000000   \n",
       "2              6704.0                   224.0          1.0   360.000000   \n",
       "3              2320.0                   290.0          1.0    54.993103   \n",
       "4              6328.0                   201.0          1.0   360.000000   \n",
       "...               ...                     ...          ...          ...   \n",
       "875282         9920.0                   124.0          1.0   360.000000   \n",
       "875283         8704.0                  1088.0          1.0   360.000000   \n",
       "875284            0.0                     5.0          1.0    36.000000   \n",
       "875285            8.0                     2.0          1.0    36.000000   \n",
       "875286        15104.0                   128.0          1.0   360.000000   \n",
       "\n",
       "        lustre_stripe_size  lustre_stripe_offset  lustre_stripe_width  \\\n",
       "0                1048576.0                   0.0                  4.0   \n",
       "1                1048576.0                   0.0                176.0   \n",
       "2                1048576.0                   0.0                224.0   \n",
       "3                1048576.0                   0.0                290.0   \n",
       "4                1048576.0                   0.0                201.0   \n",
       "...                    ...                   ...                  ...   \n",
       "875282           1048576.0                   0.0                124.0   \n",
       "875283           1048576.0                   0.0               1088.0   \n",
       "875284           1048576.0                   0.0                  5.0   \n",
       "875285           1048576.0                   0.0                  2.0   \n",
       "875286           1048576.0                   0.0                128.0   \n",
       "\n",
       "        lustre_number_of_osts      jobid  \\\n",
       "0                         4.0  9042391.0   \n",
       "1                       147.0  9868242.0   \n",
       "2                       186.0  9029683.0   \n",
       "3                        52.0  9165166.0   \n",
       "4                       158.0  8269019.0   \n",
       "...                       ...        ...   \n",
       "875282                  110.0  7759072.0   \n",
       "875283                  359.0  7275819.0   \n",
       "875284                    5.0  8305062.0   \n",
       "875285                    1.0    31428.0   \n",
       "875286                  118.0  7525550.0   \n",
       "\n",
       "                                                     path  \n",
       "0       ../results/USER375904U_sss2_id9042391_9-26-414...  \n",
       "1       ../results/USER382862U_SpEC_id9868242_4-29-741...  \n",
       "2       ../results/USER264205U_SpEC_id9029683_9-20-580...  \n",
       "3       ../results/USER341996U_CCTM_v52_Linux2_x86_64i...  \n",
       "4       ../results/USER141481U_SpEC_id8269019_3-3-4542...  \n",
       "...                                                   ...  \n",
       "875282  ../results/USER336298U_SpEC_id7759072_10-30-62...  \n",
       "875283  ../results/USER331454U_pw.x_id7275819_8-2-4790...  \n",
       "875284  ../results/USER371284U_sss2_id8305062_3-14-211...  \n",
       "875285  ../results/USER303986U_testnodes_opt.x_id31428...  \n",
       "875286  ../results/USER382862U_SpEC_id7525550_9-24-380...  \n",
       "\n",
       "[875287 rows x 50 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df = df.dropna(axis=0, how='any')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#files\n",
    "f['p_files'] = df['posix_number_of_files']  + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0               0.0\n",
       "1          880136.0\n",
       "2         2379598.0\n",
       "3         8903411.0\n",
       "4         7846387.0\n",
       "            ...    \n",
       "875282    2234152.0\n",
       "875283     197651.0\n",
       "875284          0.0\n",
       "875285          0.0\n",
       "875286    6065006.0\n",
       "Name: p_accesses, Length: 875287, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#accesses\n",
    "df['p_accesses'] = df['posix_reads'] + df['posix_writes']\n",
    "f['p_accesses'] = df['p_accesses']\n",
    "\n",
    "f['p_accesses']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#bytes\n",
    "f['log10_p_bytes'] = df['posix_bytes_read'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['log10_p_opens'] = df['posix_opens']\n",
    "f['log10_p_seeks'] = df['posix_seeks']\n",
    "f['log10_p_stats'] = df['posix_stats']\n",
    "f['log10_p_mode'] = df['posix_renamed_mode']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dave/opt/anaconda3/lib/python3.8/site-packages/pandas/core/arraylike.py:358: RuntimeWarning: divide by zero encountered in log10\n",
      "  result = getattr(ufunc, method)(*inputs, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "f['log10_p_nprocs'] = np.log10(df['nprocs'])\n",
    "f['log10_p_falign'] = np.log10(df['posix_f_align'])\n",
    "f['log10_p_malign'] = np.log10(df['posix_m_align'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "f['p_reads'] = df['posix_reads'] \n",
    "f['p_writes'] = df['posix_writes'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['histogram_r_sum'] = df['posix_bytes_read_100'] + df['posix_bytes_read_1K'] + \\\n",
    "df['posix_bytes_read_4M'] + df['posix_bytes_read_10M'] + df['posix_bytes_read_100M'] +\\\n",
    "df['posix_bytes_read_1G'] + df['posix_bytes_read_PLUS']\n",
    "\n",
    "f['perc_p_bytes_read_100'] = df['posix_bytes_read_100']\n",
    "f['perc_p_bytes_read_1K'] = df['posix_bytes_read_1K'] \n",
    "f['perc_p_bytes_read_10K'] = df['posix_bytes_read_10K']\n",
    "f['perc_p_bytes_read_100K'] = df['posix_bytes_read_100K']\n",
    "f['perc_p_bytes_read_1M'] = df['posix_bytes_read_1lM']\n",
    "f['perc_p_bytes_read_4M'] = df['posix_bytes_read_4M']\n",
    "f['perc_p_bytes_read_10M'] = df['posix_bytes_read_10M']\n",
    "f['perc_p_bytes_read_100M'] = df['posix_bytes_read_100M']\n",
    "f['perc_p_bytes_read_1G'] = df['posix_bytes_read_1G']\n",
    "f['perc_p_bytes_read_PLUS'] = df['posix_bytes_read_PLUS']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['histogram_w_sum'] = df['posix_bytes_write_100'] + df['posix_bytes_write_1K'] + \\\n",
    "df['posix_bytes_write_10K'] + df['posix_bytes_write_100K'] + df['posix_bytes_write_1M'] +\\\n",
    "df['posix_bytes_write_4M'] + df['posix_bytes_write_10M'] + df['posix_bytes_write_100M'] +\\\n",
    "df['posix_bytes_write_1G'] + df['posix_bytes_write_PLUS']\n",
    "\n",
    "f['perc_p_bytes_write_100'] = df['posix_bytes_write_100']\n",
    "f['perc_p_bytes_write_1K'] = df['posix_bytes_write_1K']\n",
    "f['perc_p_bytes_write_10K'] = df['posix_bytes_write_10K']\n",
    "f['perc_p_bytes_write_100K'] = df['posix_bytes_write_100K']\n",
    "f['perc_p_bytes_write_1M'] = df['posix_bytes_write_1M']\n",
    "f['perc_p_bytes_write_4M'] = df['posix_bytes_write_4M']\n",
    "f['perc_p_bytes_write_10M'] = df['posix_bytes_write_10M']\n",
    "f['perc_p_bytes_write_100M'] = df['posix_bytes_write_100M']\n",
    "f['perc_p_bytes_write_1G'] = df['posix_bytes_write_1G']\n",
    "f['perc_p_bytes_write_PLUS'] = df['posix_bytes_write_PLUS'] \n",
    "\n",
    "\n",
    "f = f.replace(-np.inf, -1)\n",
    "f = f.replace(np.nan, 0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['time'] = df['posix_write_time'] + df['posix_read_time'] + df['posix_meta_time']\n",
    "df['bytes'] = df['posix_bytes_read'] + df['posix_bytes_write']\n",
    "\n",
    "t = pd.DataFrame()\n",
    "f['throughput'] = df['bytes'] / df['time']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "p_files                    1.340070e+05\n",
       "p_accesses                 2.251942e+10\n",
       "log10_p_bytes              3.038456e+14\n",
       "log10_p_opens              5.531094e+08\n",
       "log10_p_seeks              1.445220e+10\n",
       "log10_p_stats              6.522921e+07\n",
       "log10_p_mode               5.337293e+07\n",
       "log10_p_nprocs             5.719570e+00\n",
       "log10_p_falign             1.115306e+01\n",
       "log10_p_malign             6.035555e+00\n",
       "p_reads                    2.237846e+10\n",
       "p_writes                   1.302770e+10\n",
       "perc_p_bytes_read_100      5.221517e+08\n",
       "perc_p_bytes_read_1K       2.074657e+10\n",
       "perc_p_bytes_read_10K      1.536278e+09\n",
       "perc_p_bytes_read_100K     1.515506e+08\n",
       "perc_p_bytes_read_1M       4.044503e+08\n",
       "perc_p_bytes_read_4M       6.561462e+07\n",
       "perc_p_bytes_read_10M      2.083200e+06\n",
       "perc_p_bytes_read_100M     2.872090e+05\n",
       "perc_p_bytes_read_1G       1.792000e+06\n",
       "perc_p_bytes_write_100     1.302770e+10\n",
       "perc_p_bytes_write_1K      2.852127e+09\n",
       "perc_p_bytes_write_10K     3.867477e+08\n",
       "perc_p_bytes_write_100K    8.347452e+07\n",
       "perc_p_bytes_write_1M      1.357245e+07\n",
       "perc_p_bytes_write_4M      3.839488e+06\n",
       "perc_p_bytes_write_10M     6.190660e+05\n",
       "perc_p_bytes_write_100M    1.249280e+06\n",
       "perc_p_bytes_write_1G      1.937500e+04\n",
       "throughput                 2.344536e+09\n",
       "dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#delete columns with all zeros\n",
    "f = f.loc[:, (f != 0).any(axis=0)]\n",
    "\n",
    "#remove infinite values\n",
    "f = f.replace([np.inf, -np.inf], np.nan).dropna(axis=0)\n",
    "\n",
    "f.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>p_files</th>\n",
       "      <th>p_accesses</th>\n",
       "      <th>log10_p_bytes</th>\n",
       "      <th>log10_p_opens</th>\n",
       "      <th>log10_p_seeks</th>\n",
       "      <th>log10_p_stats</th>\n",
       "      <th>log10_p_mode</th>\n",
       "      <th>log10_p_nprocs</th>\n",
       "      <th>log10_p_falign</th>\n",
       "      <th>log10_p_malign</th>\n",
       "      <th>...</th>\n",
       "      <th>perc_p_bytes_read_1G</th>\n",
       "      <th>perc_p_bytes_write_100</th>\n",
       "      <th>perc_p_bytes_write_1K</th>\n",
       "      <th>perc_p_bytes_write_10K</th>\n",
       "      <th>perc_p_bytes_write_100K</th>\n",
       "      <th>perc_p_bytes_write_1M</th>\n",
       "      <th>perc_p_bytes_write_4M</th>\n",
       "      <th>perc_p_bytes_write_10M</th>\n",
       "      <th>perc_p_bytes_write_100M</th>\n",
       "      <th>perc_p_bytes_write_1G</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>800.0</td>\n",
       "      <td>880136.0</td>\n",
       "      <td>2.390891e+10</td>\n",
       "      <td>8858.0</td>\n",
       "      <td>319241.0</td>\n",
       "      <td>34901.0</td>\n",
       "      <td>37230.0</td>\n",
       "      <td>1.681241</td>\n",
       "      <td>9.254357</td>\n",
       "      <td>4.136847</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>202780.0</td>\n",
       "      <td>16078.0</td>\n",
       "      <td>21989.0</td>\n",
       "      <td>3156.0</td>\n",
       "      <td>1061.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>361.0</td>\n",
       "      <td>2379598.0</td>\n",
       "      <td>5.019637e+10</td>\n",
       "      <td>62398.0</td>\n",
       "      <td>1107764.0</td>\n",
       "      <td>270222.0</td>\n",
       "      <td>27594.0</td>\n",
       "      <td>1.681241</td>\n",
       "      <td>8.943844</td>\n",
       "      <td>3.826334</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18852.0</td>\n",
       "      <td>264906.0</td>\n",
       "      <td>22246.0</td>\n",
       "      <td>4282.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>291.0</td>\n",
       "      <td>8903411.0</td>\n",
       "      <td>5.488943e+12</td>\n",
       "      <td>8711.0</td>\n",
       "      <td>2010273.0</td>\n",
       "      <td>28432.0</td>\n",
       "      <td>115194.0</td>\n",
       "      <td>2.408240</td>\n",
       "      <td>8.482998</td>\n",
       "      <td>3.365488</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7302089.0</td>\n",
       "      <td>52045.0</td>\n",
       "      <td>73.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>25830.0</td>\n",
       "      <td>756.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>320.0</td>\n",
       "      <td>7846387.0</td>\n",
       "      <td>2.293203e+10</td>\n",
       "      <td>23158.0</td>\n",
       "      <td>6015926.0</td>\n",
       "      <td>400399.0</td>\n",
       "      <td>31536.0</td>\n",
       "      <td>1.681241</td>\n",
       "      <td>8.918776</td>\n",
       "      <td>3.801266</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>489873.0</td>\n",
       "      <td>2821931.0</td>\n",
       "      <td>240372.0</td>\n",
       "      <td>17983.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>145.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.505150</td>\n",
       "      <td>8.178962</td>\n",
       "      <td>3.061452</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875280</th>\n",
       "      <td>2.0</td>\n",
       "      <td>57344.0</td>\n",
       "      <td>3.006477e+10</td>\n",
       "      <td>1808.0</td>\n",
       "      <td>59128.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.049218</td>\n",
       "      <td>6.020600</td>\n",
       "      <td>0.903090</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28672.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875281</th>\n",
       "      <td>14.0</td>\n",
       "      <td>102439.0</td>\n",
       "      <td>7.467916e+08</td>\n",
       "      <td>193.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>385.0</td>\n",
       "      <td>4818.0</td>\n",
       "      <td>1.505150</td>\n",
       "      <td>7.134543</td>\n",
       "      <td>2.017033</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>10927.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875282</th>\n",
       "      <td>625.0</td>\n",
       "      <td>2234152.0</td>\n",
       "      <td>5.593977e+10</td>\n",
       "      <td>35112.0</td>\n",
       "      <td>1035457.0</td>\n",
       "      <td>159180.0</td>\n",
       "      <td>16644.0</td>\n",
       "      <td>1.505150</td>\n",
       "      <td>9.114022</td>\n",
       "      <td>3.996512</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>51807.0</td>\n",
       "      <td>397643.0</td>\n",
       "      <td>27690.0</td>\n",
       "      <td>3198.0</td>\n",
       "      <td>1207.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875283</th>\n",
       "      <td>1089.0</td>\n",
       "      <td>197651.0</td>\n",
       "      <td>1.465277e+09</td>\n",
       "      <td>2112.0</td>\n",
       "      <td>12509.0</td>\n",
       "      <td>4198.0</td>\n",
       "      <td>473916.0</td>\n",
       "      <td>2.709270</td>\n",
       "      <td>9.057229</td>\n",
       "      <td>3.939719</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13824.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>875286</th>\n",
       "      <td>583.0</td>\n",
       "      <td>6065006.0</td>\n",
       "      <td>1.077412e+11</td>\n",
       "      <td>119432.0</td>\n",
       "      <td>2750807.0</td>\n",
       "      <td>578508.0</td>\n",
       "      <td>17520.0</td>\n",
       "      <td>1.505150</td>\n",
       "      <td>9.296602</td>\n",
       "      <td>4.179092</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216962.0</td>\n",
       "      <td>1239000.0</td>\n",
       "      <td>77141.0</td>\n",
       "      <td>6985.0</td>\n",
       "      <td>272.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>708907 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        p_files  p_accesses  log10_p_bytes  log10_p_opens  log10_p_seeks  \\\n",
       "1         800.0    880136.0   2.390891e+10         8858.0       319241.0   \n",
       "2         361.0   2379598.0   5.019637e+10        62398.0      1107764.0   \n",
       "3         291.0   8903411.0   5.488943e+12         8711.0      2010273.0   \n",
       "4         320.0   7846387.0   2.293203e+10        23158.0      6015926.0   \n",
       "5         145.0         0.0   0.000000e+00            0.0            0.0   \n",
       "...         ...         ...            ...            ...            ...   \n",
       "875280      2.0     57344.0   3.006477e+10         1808.0        59128.0   \n",
       "875281     14.0    102439.0   7.467916e+08          193.0           91.0   \n",
       "875282    625.0   2234152.0   5.593977e+10        35112.0      1035457.0   \n",
       "875283   1089.0    197651.0   1.465277e+09         2112.0        12509.0   \n",
       "875286    583.0   6065006.0   1.077412e+11       119432.0      2750807.0   \n",
       "\n",
       "        log10_p_stats  log10_p_mode  log10_p_nprocs  log10_p_falign  \\\n",
       "1             34901.0       37230.0        1.681241        9.254357   \n",
       "2            270222.0       27594.0        1.681241        8.943844   \n",
       "3             28432.0      115194.0        2.408240        8.482998   \n",
       "4            400399.0       31536.0        1.681241        8.918776   \n",
       "5               176.0           0.0        1.505150        8.178962   \n",
       "...               ...           ...             ...             ...   \n",
       "875280           12.0           0.0        2.049218        6.020600   \n",
       "875281          385.0        4818.0        1.505150        7.134543   \n",
       "875282       159180.0       16644.0        1.505150        9.114022   \n",
       "875283         4198.0      473916.0        2.709270        9.057229   \n",
       "875286       578508.0       17520.0        1.505150        9.296602   \n",
       "\n",
       "        log10_p_malign  ...  perc_p_bytes_read_1G  perc_p_bytes_write_100  \\\n",
       "1             4.136847  ...                   0.0                202780.0   \n",
       "2             3.826334  ...                   0.0                 18852.0   \n",
       "3             3.365488  ...                   0.0               7302089.0   \n",
       "4             3.801266  ...                   0.0                489873.0   \n",
       "5             3.061452  ...                   0.0                     0.0   \n",
       "...                ...  ...                   ...                     ...   \n",
       "875280        0.903090  ...                   0.0                     0.0   \n",
       "875281        2.017033  ...                   0.0                    33.0   \n",
       "875282        3.996512  ...                   0.0                 51807.0   \n",
       "875283        3.939719  ...                   0.0                    28.0   \n",
       "875286        4.179092  ...                   0.0                216962.0   \n",
       "\n",
       "        perc_p_bytes_write_1K  perc_p_bytes_write_10K  \\\n",
       "1                     16078.0                 21989.0   \n",
       "2                    264906.0                 22246.0   \n",
       "3                     52045.0                    73.0   \n",
       "4                   2821931.0                240372.0   \n",
       "5                         0.0                     0.0   \n",
       "...                       ...                     ...   \n",
       "875280                    0.0                     0.0   \n",
       "875281                   23.0                 10927.0   \n",
       "875282               397643.0                 27690.0   \n",
       "875283                    0.0                    38.0   \n",
       "875286              1239000.0                 77141.0   \n",
       "\n",
       "        perc_p_bytes_write_100K  perc_p_bytes_write_1M  perc_p_bytes_write_4M  \\\n",
       "1                        3156.0                 1061.0                    0.0   \n",
       "2                        4282.0                   14.0                    0.0   \n",
       "3                           9.0                25830.0                  756.0   \n",
       "4                       17983.0                  242.0                    0.0   \n",
       "5                           0.0                    0.0                    0.0   \n",
       "...                         ...                    ...                    ...   \n",
       "875280                      0.0                28672.0                    0.0   \n",
       "875281                      0.0                    0.0                    0.0   \n",
       "875282                   3198.0                 1207.0                    0.0   \n",
       "875283                      0.0                13824.0                    0.0   \n",
       "875286                   6985.0                  272.0                    0.0   \n",
       "\n",
       "        perc_p_bytes_write_10M  perc_p_bytes_write_100M  perc_p_bytes_write_1G  \n",
       "1                          0.0                      0.0                    0.0  \n",
       "2                          0.0                      0.0                    0.0  \n",
       "3                          0.0                      0.0                    0.0  \n",
       "4                          0.0                      0.0                    0.0  \n",
       "5                          0.0                      0.0                    0.0  \n",
       "...                        ...                      ...                    ...  \n",
       "875280                     0.0                      0.0                    0.0  \n",
       "875281                     0.0                      0.0                    0.0  \n",
       "875282                     0.0                      0.0                    0.0  \n",
       "875283                     0.0                      0.0                    0.0  \n",
       "875286                     0.0                      0.0                    0.0  \n",
       "\n",
       "[708907 rows x 30 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t['throughput'] = f['throughput']\n",
    "f = f.drop(labels = 'throughput', axis = 1)\n",
    "f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "f = StandardScaler().fit_transform(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "throughput    2.344536e+09\n",
       "dtype: float64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "throughput    0.0\n",
      "dtype: float64\n",
      "throughput    2.344536e+09\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print(t.min())\n",
    "print(t.max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.metrics import mean_squared_log_error\n",
    "from sklearn.metrics import mean_absolute_percentage_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2825\n",
      "Mean True Value: \t 80247938\n",
      "Mean Absolute Error: \t 51128803\n",
      "Mean Squared Error:  8049291975754185.0\n",
      "Root Mean Squared Error:  89717846.47300772\n",
      "MAPE :9.48740052737349e+21\n",
      "R2: 0.3157999993460463\n",
      "\n",
      "3696\n",
      "Mean True Value: \t 79774347\n",
      "Mean Absolute Error: \t 50994053\n",
      "Mean Squared Error:  8398636528105234.0\n",
      "Root Mean Squared Error:  91644075.24824086\n",
      "MAPE :9.669046250027161e+21\n",
      "R2: 0.2813313403030343\n",
      "\n",
      "3650\n",
      "Mean True Value: \t 80041315\n",
      "Mean Absolute Error: \t 50781597\n",
      "Mean Squared Error:  7814574077175511.0\n",
      "Root Mean Squared Error:  88400079.62199758\n",
      "MAPE :9.067264526818159e+21\n",
      "R2: 0.32493823392307575\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LinearRegression\n",
    "rseed = 0\n",
    "t_size = 0.1\n",
    "\n",
    "for i in range(3):\n",
    "    \n",
    "    rseed = random.randint(1,10000)\n",
    "    print(rseed)\n",
    "    train_data, test_data, train_labels, test_labels = train_test_split(f,t, test_size=t_size, random_state=rseed)\n",
    "\n",
    "    reg = LinearRegression().fit(train_data, train_labels)\n",
    "\n",
    "    predicted_labels = reg.predict(test_data)\n",
    "\n",
    "    print(\"Mean True Value: \\t\",int(test_labels.mean() ))\n",
    "    print(\"Mean Absolute Error: \\t\", int(mean_absolute_error(test_labels, predicted_labels) ))\n",
    "    print(\"Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels) )\n",
    "    print(\"Root Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels, squared = False) )\n",
    "    print(\"MAPE :\" + str(mean_absolute_percentage_error( test_labels, predicted_labels )))\n",
    "    print(\"R2: \" + str(r2_score(test_labels,predicted_labels)) + \"\\n\")\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4291\n",
      "Mean True Value: \t 79671972\n",
      "Mean Absolute Error: \t 15775270\n",
      "Mean Squared Error:  1182917720629453.5\n",
      "Root Mean Squared Error:  34393570.92000558\n",
      "MAPE :5.928271388112404e+19\n",
      "R2: 0.8964572242755775\n",
      "\n",
      "5005\n",
      "Mean True Value: \t 79970693\n",
      "Mean Absolute Error: \t 15753294\n",
      "Mean Squared Error:  1171118466969232.2\n",
      "Root Mean Squared Error:  34221608.18794512\n",
      "MAPE :7.529020324371345e+19\n",
      "R2: 0.9001025528311228\n",
      "\n",
      "1952\n",
      "Mean True Value: \t 80010257\n",
      "Mean Absolute Error: \t 15694609\n",
      "Mean Squared Error:  1167859298304569.8\n",
      "Root Mean Squared Error:  34173956.43329244\n",
      "MAPE :7.649769608686325e+19\n",
      "R2: 0.9004918036900794\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#EXTREME GRADIENT BOOST\n",
    "import xgboost as xg\n",
    "for i in range(3):\n",
    "    rseed = random.randint(1,10000)\n",
    "    print(rseed)\n",
    "\n",
    "    train_data, test_data, train_labels, test_labels = train_test_split(f,t, test_size=t_size, random_state=rseed)\n",
    "    xgb_r = xg.XGBRegressor(n_estimators = 1000, seed = 123)\n",
    "    xgb_r.fit(train_data, train_labels)\n",
    "    predicted_labels = xgb_r.predict(test_data)\n",
    "\n",
    "    print(\"Mean True Value: \\t\",int(test_labels.mean() ))\n",
    "    print(\"Mean Absolute Error: \\t\", int(mean_absolute_error(test_labels, predicted_labels) ))\n",
    "    print(\"Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels) )\n",
    "    print(\"Root Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels, squared = False) )\n",
    "    print(\"MAPE :\" + str(mean_absolute_percentage_error( test_labels, predicted_labels )))\n",
    "    print(\"R2: \" + str(r2_score(test_labels,predicted_labels)) + \"\\n\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9163\n",
      "Mean True Value: \t 79868453\n",
      "Mean Absolute Error: \t 33248390\n",
      "Mean Squared Error:  3218231073977619.5\n",
      "Root Mean Squared Error:  56729455.08267834\n",
      "MAPE :7.696365899110741e+21\n",
      "R2: 0.721500661764336\n",
      "\n",
      "3420\n",
      "Mean True Value: \t 79722234\n",
      "Mean Absolute Error: \t 33700230\n",
      "Mean Squared Error:  3251758114891718.5\n",
      "Root Mean Squared Error:  57024188.8578147\n",
      "MAPE :7.824038340112399e+21\n",
      "R2: 0.7186109537851515\n",
      "\n",
      "5522\n",
      "Mean True Value: \t 80104539\n",
      "Mean Absolute Error: \t 33107125\n",
      "Mean Squared Error:  3156004880382560.0\n",
      "Root Mean Squared Error:  56178331.05729076\n",
      "MAPE :7.758545262697204e+21\n",
      "R2: 0.7284183849924653\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "\n",
    "for i in range(3):\n",
    "    rseed = random.randint(1,10000)\n",
    "    print(rseed)\n",
    "\n",
    "    train_data, test_data, train_labels, test_labels = train_test_split(f,t, test_size= t_size, random_state=rseed)\n",
    "    \n",
    "    \n",
    "    reg = DecisionTreeRegressor(max_depth = 4)   \n",
    "    \n",
    "\n",
    "    reg.fit(train_data, train_labels)\n",
    "    predicted_labels = reg.predict(test_data)\n",
    "\n",
    "\n",
    "    print(\"Mean True Value: \\t\",int(test_labels.mean() ))\n",
    "    print(\"Mean Absolute Error: \\t\", int(mean_absolute_error(test_labels, predicted_labels) ))\n",
    "    print(\"Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels) )\n",
    "    print(\"Root Mean Squared Error: \", mean_squared_error(test_labels, predicted_labels, squared = False) )\n",
    "    print(\"MAPE :\" + str(mean_absolute_percentage_error( test_labels, predicted_labels )))\n",
    "    print(\"R2: \" + str(r2_score(test_labels,predicted_labels)) + \"\\n\")\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'x' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-24-afbf8c2948db>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msubplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'bo-'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Using 1D Cubic Spline Interpolation'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'x' is not defined"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAACGCAYAAADQHI0rAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAALJElEQVR4nO3dX4hc533G8e/TlQWNa+IkXrtBshq1qHVViMGZKm6TNnaLU8k0iIAv5IYYTEC4jUvpRYnphXPRm5bclLROhDAi5CLWRWMnKsiWDaF1qOtUq+I/khOHrZLGiwL+i0OdUiPn14s5QsN613u0Ozuz2ff7gWHnnPd9Z3/zsnuePWfnnJOqQpLUrl+YdgGSpOkyCCSpcQaBJDXOIJCkxhkEktQ4g0CSGrdiECQ5kuTFJKeXaU+SLyaZT/JMkhtG2vYmeb5ru2echUuSxqPPHsFXgL3v0L4P2NU9DgJfBkgyA9zXte8Gbk+yey3FSpLGb8UgqKrHgVffoct+4Ks19CRwZZL3A3uA+ao6W1VvAke7vpKkDWQc/yPYBrwwsrzQrVtuvSRpA9kyhtfIEuvqHdYv/SLJQYaHlrj88ss/dN11142hNElqw6lTp16uqtnVjB1HECwA144sbwfOAVuXWb+kqjoMHAYYDAY1Nzc3htIkqQ1J/nu1Y8dxaOgYcEf36aEbgder6sfASWBXkp1JtgIHur6SpA1kxT2CJA8ANwFXJVkAPg9cBlBVh4DjwK3APPBT4M6u7XySu4ETwAxwpKrOrMN7kCStwYpBUFW3r9BewGeXaTvOMCgkSRuUZxZLUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhrXKwiS7E3yfJL5JPcs0f5XSZ7qHqeTvJXkvV3bD5M827V5I2JJ2mD63KpyBrgPuIXhjepPJjlWVc9d6FNVXwC+0PX/BPCXVfXqyMvcXFUvj7VySdJY9Nkj2APMV9XZqnoTOArsf4f+twMPjKM4SdL66xME24AXRpYXunVvk+RdwF7g6yOrC3g0yakkB1dbqCRpfax4aAjIEutqmb6fAP5t0WGhj1TVuSRXA48l+V5VPf62bzIMiYMAO3bs6FGWJGkc+uwRLADXjixvB84t0/cAiw4LVdW57uuLwEMMDzW9TVUdrqpBVQ1mZ2d7lCVJGoc+QXAS2JVkZ5KtDDf2xxZ3SvJu4GPAN0fWXZ7kigvPgY8Dp8dRuCRpPFY8NFRV55PcDZwAZoAjVXUmyV1d+6Gu6yeBR6vqjZHh1wAPJbnwvb5WVY+M8w1IktYmVcsd7p+ewWBQc3OeciBJfSU5VVWD1Yz1zGJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuN6BUGSvUmeTzKf5J4l2m9K8nqSp7rHvX3HSpKma8VbVSaZAe4DbmF4I/uTSY5V1XOLun67qv54lWMlSVPSZ49gDzBfVWer6k3gKLC/5+uvZawkaQL6BME24IWR5YVu3WK/k+TpJA8n+a1LHEuSg0nmksy99NJLPcqSJI1DnyDIEusW3/H+P4FfqarrgX8AvnEJY4crqw5X1aCqBrOzsz3KkiSNQ58gWACuHVneDpwb7VBVP6mq/+meHwcuS3JVn7GSpOnqEwQngV1JdibZChwAjo12SPLLSdI939O97it9xkqSpmvFTw1V1fkkdwMngBngSFWdSXJX134IuA340yTngf8FDlRVAUuOXaf3IklahQy31xvLYDCoubm5aZchST83kpyqqsFqxnpmsSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY3rFQRJ9iZ5Psl8knuWaP9Ukme6xxNJrh9p+2GSZ5M8lcSbDEjSBrPiHcqSzAD3AbcwvAfxySTHquq5kW4/AD5WVa8l2QccBj480n5zVb08xrolSWPSZ49gDzBfVWer6k3gKLB/tENVPVFVr3WLTzK8Sb0k6edAnyDYBrwwsrzQrVvOZ4CHR5YLeDTJqSQHL71ESdJ6WvHQEJAl1i15o+MkNzMMgo+OrP5IVZ1LcjXwWJLvVdXjS4w9CBwE2LFjR4+yJEnj0GePYAG4dmR5O3BucackHwTuB/ZX1SsX1lfVue7ri8BDDA81vU1VHa6qQVUNZmdn+78DSdKa9AmCk8CuJDuTbAUOAMdGOyTZATwIfLqqvj+y/vIkV1x4DnwcOD2u4iVJa7fioaGqOp/kbuAEMAMcqaozSe7q2g8B9wLvA76UBOB8VQ2Aa4CHunVbgK9V1SPr8k4kSauSqiUP90/VYDCouTlPOZCkvpKc6v4Av2SeWSxJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJalyvIEiyN8nzSeaT3LNEe5J8sWt/JskNfcdKkqZrxSBIMgPcB+wDdgO3J9m9qNs+YFf3OAh8+RLGSpKmqM8ewR5gvqrOVtWbwFFg/6I++4Gv1tCTwJVJ3t9zrCRpivoEwTbghZHlhW5dnz59xkqSpmhLjz5ZYt3iO94v16fP2OELJAcZHlYC+L8kp3vU1oKrgJenXcQG4Dxc5Fxc5Fxc9BurHdgnCBaAa0eWtwPnevbZ2mMsAFV1GDgMkGSuqgY9atv0nIsh5+Ei5+Ii5+KiJHOrHdvn0NBJYFeSnUm2AgeAY4v6HAPu6D49dCPwelX9uOdYSdIUrbhHUFXnk9wNnABmgCNVdSbJXV37IeA4cCswD/wUuPOdxq7LO5EkrUqfQ0NU1XGGG/vRdYdGnhfw2b5jezh8if03M+diyHm4yLm4yLm4aNVzkeE2XJLUKi8xIUmNm1oQrOWyFZtNj7n4VDcHzyR5Isn106hzEvpekiTJbyd5K8ltk6xvkvrMRZKbkjyV5EySf510jZPS43fk3Un+OcnT3VzcOY0611uSI0leXO7j9aveblbVxB8M/3H8X8CvMvyI6dPA7kV9bgUeZnguwo3Ad6ZR6waZi98F3tM939fyXIz0+xbD/z3dNu26p/hzcSXwHLCjW7562nVPcS7+Gvi77vks8Cqwddq1r8Nc/D5wA3B6mfZVbTentUewlstWbDYrzkVVPVFVr3WLTzI8H2Mz6ntJkj8Hvg68OMniJqzPXPwJ8GBV/QigqjbrfPSZiwKuSBLglxgGwfnJlrn+qupxhu9tOavabk4rCNZy2YrN5lLf52cYJv5mtOJcJNkGfBI4xObW5+fi14H3JPmXJKeS3DGx6iarz1z8I/CbDE9YfRb4i6r62WTK21BWtd3s9fHRdbCWy1ZsNpdyGY6bGQbBR9e1ounpMxd/D3yuqt4a/vG3afWZiy3Ah4A/BH4R+PckT1bV99e7uAnrMxd/BDwF/AHwa8BjSb5dVT9Z59o2mlVtN6cVBGu5bMVm0+t9JvkgcD+wr6pemVBtk9ZnLgbA0S4ErgJuTXK+qr4xkQonp+/vyMtV9QbwRpLHgeuBzRYEfebiTuBva3igfD7JD4DrgP+YTIkbxqq2m9M6NLSWy1ZsNivORZIdwIPApzfhX3ujVpyLqtpZVR+oqg8A/wT82SYMAej3O/JN4PeSbEnyLuDDwHcnXOck9JmLHzHcMyLJNQwvwHZ2olVuDKvabk5lj6DWcNmKzabnXNwLvA/4UveX8PnahBfa6jkXTegzF1X13SSPAM8APwPur6pNd9Xenj8XfwN8JcmzDA+PfK6qNt1VSZM8ANwEXJVkAfg8cBmsbbvpmcWS1DjPLJakxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ17v8B+hrXNvxq5OsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "t_size = 0.2\n",
    "rseed = 1\n",
    "train_data, test_data, train_labels, test_labels = train_test_split(f,t, test_size=t_size, random_state=rseed)\n",
    "\n",
    "new_length = 500\n",
    "\n",
    "new_x = np.linspace(train_data.min(), train_data.max(), new_length)\n",
    "print((train_data.shape))\n",
    "print((train_labels.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}