Part2_Imputation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8b62d5bc",
   "metadata": {},
   "source": [
    "# Implementation of all the Imputation Techniques including FPCKNN"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3861ff2c",
   "metadata": {},
   "source": [
    "## Doing Imputation on Multivariate Time Series Data (3D)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "910f9d22",
   "metadata": {},
   "source": [
    "### Reading the Data built by the Previous Notebooks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cc360cca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading the Raw Data\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/1_Raw/\"\n",
    "raw_data = []\n",
    "\n",
    "num_partitions = 5\n",
    "\n",
    "for i in range(0,num_partitions):\n",
    "# Load the array with Pickle\n",
    "    with open(data_dir + \"Partition\" + str(i+1) + \".pkl\", 'rb') as f:\n",
    "        raw_data.append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e526095",
   "metadata": {},
   "source": [
    "### Nextvalue Imputation Technique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7e655130",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "def next_or_previous_value_imputation(vector):\n",
    "    # Convert the vector to a Pandas Series\n",
    "    series = pd.Series(vector)\n",
    "    \n",
    "    # Perform next value imputation (backfill)\n",
    "    next_imputed_series = series[::-1].fillna(method='ffill')[::-1]\n",
    "\n",
    "    # Perform previous value imputation (forward fill) only for NaNs at the end\n",
    "    previous_imputed_series = next_imputed_series.fillna(method='ffill')\n",
    "\n",
    "    return previous_imputed_series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4a3ee59a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "73492it [01:58, 622.50it/s]\n",
      "88557it [02:28, 596.86it/s]\n",
      "42510it [01:13, 575.67it/s]\n",
      "51261it [01:27, 585.09it/s]\n",
      "75365it [02:12, 566.97it/s]\n"
     ]
    }
   ],
   "source": [
    "# Inter Column and Between Instance Imputation\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/3_2_BaselineImputation/\"\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "k = 100\n",
    "number_of_partitions = 5\n",
    "for i in range(0,number_of_partitions):\n",
    "    new_partition = np.zeros((60,25,np.array(raw_data[i]).shape[2]))\n",
    "    new_partition = np.array(raw_data[i])\n",
    "    \n",
    "    with tqdm(np.array(raw_data[i]).shape[2]) as pbar:\n",
    "        for j in range(0,np.array(raw_data[i]).shape[2]):\n",
    "            new_column = np.zeros((60,25))  \n",
    "            new_column = new_partition[:,:,j]\n",
    "            new_column[new_column == 0.0] = np.nan\n",
    "            for m in range(0,24):\n",
    "\n",
    "                if np.isnan(new_column[:,m+1]).all():\n",
    "                    new_column[:,m+1] = np.ones(60) \n",
    "                else:\n",
    "                    new_column[:,m+1] = next_or_previous_value_imputation(new_column[:,m+1])\n",
    "                 \n",
    "                \n",
    "            new_partition[:,:,j] = new_column\n",
    "\n",
    "            pbar.update(1)\n",
    "\n",
    "        with open(data_dir + \"Partition\" + str(i+1) + \"_NextvalueImputation\" + \".pkl\", 'wb') as f:\n",
    "            pickle.dump(new_partition, f)\n",
    "            \n",
    "            \n",
    "# Between Instance Imputation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a96646f2",
   "metadata": {},
   "source": [
    "### Mean Imputation Technique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9796ba23",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "73492it [00:09, 8046.90it/s]\n",
      "88557it [00:10, 8127.42it/s]\n",
      "42510it [00:05, 7531.84it/s]\n",
      "51261it [00:06, 7532.10it/s]\n",
      "75365it [00:09, 7755.86it/s]\n"
     ]
    }
   ],
   "source": [
    "# Inter Column and Between Instance Imputation\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/3_2_BaselineImputation/\"\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "number_of_partitions = 5\n",
    "for i in range(0,number_of_partitions):\n",
    "    new_partition = np.zeros((60,25,np.array(raw_data[i]).shape[2]))\n",
    "    new_partition = np.array(raw_data[i])\n",
    "    \n",
    "    with tqdm(np.array(raw_data[i]).shape[2]) as pbar:\n",
    "        for j in range(0,np.array(raw_data[i]).shape[2]):\n",
    "            new_column = np.zeros((60,25))  \n",
    "            new_column = new_partition[:,:,j]\n",
    "            new_column[new_column == 0.0] = np.nan\n",
    "            for m in range(0,24):\n",
    "\n",
    "                if np.isnan(new_column[:,m+1]).all():\n",
    "                    new_column[:,m+1] = np.ones(60) \n",
    "                else:\n",
    "                    mean_non_zero = np.mean(new_column[:,m+1][~np.isnan(new_column[:,m+1])])\n",
    "                    new_column[:,m+1][np.isnan(new_column[:,m+1])] = mean_non_zero\n",
    "                 \n",
    "                \n",
    "            new_partition[:,:,j] = new_column\n",
    "\n",
    "            pbar.update(1)\n",
    "\n",
    "        with open(data_dir + \"Partition\" + str(i+1) + \"_MeanImputation\" + \".pkl\", 'wb') as f:\n",
    "            pickle.dump(new_partition, f)\n",
    "            \n",
    "            \n",
    "# Between Instance Imputation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a3474308",
   "metadata": {},
   "source": [
    "# FPCKNN Imputation Technique (our Novelty)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "faf9aa5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Correlation Function\n",
    "from scipy.stats import pearsonr\n",
    "\n",
    "def mvts_pearson_correlation(X, Y):\n",
    "    if X.shape != Y.shape:\n",
    "        raise ValueError(\"Input arrays X and Y must have the same shape.\")\n",
    "\n",
    "    # Calculate the Pearson correlation coefficient between X and Y\n",
    "    X = X.T.flatten()\n",
    "    Y = Y.T.flatten()\n",
    "    try:\n",
    "        correlation_coefficient , p = pearsonr(X, Y)\n",
    "        if np.isnan(correlation_coefficient):\n",
    "            correlation_coefficient = -1.0\n",
    "    except:\n",
    "        correlation_coefficient = -1.0\n",
    "    \n",
    "    return correlation_coefficient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "29f04790",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "73492it [14:03, 87.11it/s] \n",
      "88557it [17:51, 82.66it/s] \n",
      "42510it [08:15, 85.86it/s] \n",
      "51261it [10:58, 77.81it/s]\n",
      "75365it [16:19, 76.97it/s] \n"
     ]
    }
   ],
   "source": [
    "# Inter Column and Between Instance Imputation\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "from sklearn.impute import KNNImputer\n",
    "imputer = KNNImputer(n_neighbors=10)\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/3_KnnImputation/\"\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "k_count = 50\n",
    "k = 50\n",
    "number_of_partitions = 5\n",
    "num_attributes = 25\n",
    "num_timestamps = 60\n",
    "\n",
    "for i in range(0,number_of_partitions):\n",
    "    new_partition = np.zeros((num_timestamps,num_attributes,np.array(raw_data[i]).shape[2]))\n",
    "    new_partition = np.array(raw_data[i])\n",
    "    \n",
    "    with tqdm(new_partition.shape[2]) as pbar:\n",
    "        for j in range(0,new_partition.shape[2]):\n",
    "            new_column = np.zeros((num_timestamps,num_attributes))  \n",
    "            new_column = new_partition[:,:,j]\n",
    "            \n",
    "            \n",
    "            \n",
    "            new_column[new_column == 0.] = np.nan\n",
    "            \n",
    "            if np.isnan(new_column[:,1:25]).all():\n",
    "                new_column = new_partition[:,:,j-1]\n",
    "            \n",
    "            nan_index = []\n",
    "            if np.isnan(new_column).any():\n",
    "                for m in range(0,num_attributes-1):\n",
    "                    if np.isnan(new_column[:,m+1]).any():\n",
    "                        nan_index.append(m+1)\n",
    "                \n",
    "                if j < k_count:\n",
    "                    if j == 0:\n",
    "                        k = 1\n",
    "                    else:\n",
    "                        k = j\n",
    "                else:\n",
    "                    k = k_count\n",
    "                correlation_coefficient = np.full(k, -2.0)\n",
    "                the_X = new_column\n",
    "                \n",
    "                the_X = np.nan_to_num(the_X, nan=0.0)\n",
    "                \n",
    "                for n in range(0, k):\n",
    "                    the_Y = new_partition[:,:,j-n-1]\n",
    "                    correlation_coefficient[n] = mvts_pearson_correlation(the_X[:,1:25], the_Y[:,1:25])\n",
    "                    \n",
    "                for m in range(0,num_attributes-1):\n",
    "\n",
    "                    if np.isnan(new_column[:,m+1]).all():\n",
    "                        \n",
    "                        indices_of_largest = np.where(correlation_coefficient == np.max(correlation_coefficient))\n",
    "                        first_occurrence_index = indices_of_largest[0][0]\n",
    "                        new_column[:,m+1] = new_partition[:,m+1,j-first_occurrence_index-1]\n",
    "\n",
    "                    else:\n",
    "                        if j>=2:\n",
    "\n",
    "                            sorted_indices = np.argsort(correlation_coefficient)[::-1]  # Sort in descending order\n",
    "                            largest_index = sorted_indices[0]  # Index of the largest item\n",
    "                            second_largest_index = sorted_indices[1]\n",
    "\n",
    "                            new_2d = [new_partition[:,m+1,j-second_largest_index-1], new_partition[:,m+1,j-largest_index-1], new_column[:,m+1]]\n",
    "                            new_column[:,m+1] = imputer.fit_transform(new_2d)[2,:]\n",
    "                        else:\n",
    "                            new_2d = new_column[:,m+1].reshape(-1, 1)\n",
    "                            new_column[:,m+1] = imputer.fit_transform(new_2d)[:,0]\n",
    "                \n",
    "                \n",
    "            new_partition[:,:,j] = new_column\n",
    "\n",
    "            pbar.update(1)\n",
    "\n",
    "        with open(data_dir + \"Partition\" + str(i+1) + \"_KnnImputation\" + \".pkl\", 'wb') as f:\n",
    "            pickle.dump(new_partition, f)\n",
    "            \n",
    "            \n",
    "# Between Instance Imputation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c1e4c51",
   "metadata": {},
   "source": [
    "### Missing Value Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "65ff544c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/3_KnnImputation/\"\n",
    "imputed_data = []\n",
    "\n",
    "number_of_partitions = 5\n",
    "for i in range(1,number_of_partitions +1):\n",
    "# Load the array with Pickle\n",
    "    with open(data_dir + \"Partition\" + str(i) + \"_KnnImputation\" + \".pkl\", 'rb') as f:\n",
    "        imputed_data.append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "93d53c53",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_missing_values(data, start_partition, end_partition):\n",
    "    abt_header = ['Timestamp', 'R_VALUE','TOTUSJH','TOTBSQ','TOTPOT','TOTUSJZ','ABSNJZH','SAVNCPP',\n",
    "                               'USFLUX','TOTFZ','MEANPOT', 'EPSX', 'EPSY','EPSZ','MEANSHR','SHRGT45','MEANGAM',\n",
    "                                  'MEANGBT','MEANGBZ','MEANGBH','MEANJZH','TOTFY','MEANJZD','MEANALP','TOTFX']\n",
    "    num_columns = 25\n",
    "    num_timestamps = 60\n",
    "    num_partitions = 5\n",
    "    null_count = [0,0,0,0,0]\n",
    "    non_null_count = [0,0,0,0,0]\n",
    "    null_count_per_feature = np.zeros((num_partitions,num_columns), dtype=int)\n",
    "\n",
    "    for i in range(start_partition-1, end_partition):\n",
    "        partition = np.array(data[i])\n",
    "\n",
    "        for j in range(0,partition.shape[2]):\n",
    "            mvts = partition[:,:, j]\n",
    "            for m in range(0,num_columns):\n",
    "                for n in range (0,num_timestamps):\n",
    "                    if (mvts[n,m] == 0.0 or np.isnan(mvts[n,m]) or np.isinf(mvts[n,m])):\n",
    "                        null_count[i] += 1\n",
    "                        null_count_per_feature[i,m] += 1\n",
    "                    else:\n",
    "                        non_null_count[i] += 1\n",
    "\n",
    "        print(\"Partition\" + str(i+1) + \":\")\n",
    "        print(\"null counts in P\" + str(i+1) + \": \" + str(null_count[i]))\n",
    "        print(\"non-null counts in P\"+ str(i+1) + \": \" + str(non_null_count[i]))\n",
    "        for x in range(0,num_columns):\n",
    "            print(abt_header[x] + \": \" + str(null_count_per_feature[i,x]))\n",
    "\n",
    "        print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c4b3e59b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_values(data, start_partition, end_partition):\n",
    "    abt_header = ['Timestamp', 'R_VALUE','TOTUSJH','TOTBSQ','TOTPOT','TOTUSJZ','ABSNJZH','SAVNCPP',\n",
    "                               'USFLUX','TOTFZ','MEANPOT', 'EPSX', 'EPSY','EPSZ','MEANSHR','SHRGT45','MEANGAM',\n",
    "                                  'MEANGBT','MEANGBZ','MEANGBH','MEANJZH','TOTFY','MEANJZD','MEANALP','TOTFX']\n",
    "    num_columns = 2\n",
    "    num_partitions = 5\n",
    "\n",
    "    for i in range(start_partition-1, end_partition):\n",
    "        partition = np.array(data[i])\n",
    "        print('Partition: ' + str(i+1))\n",
    "        for j in range(1,partition.shape[1]):\n",
    "            mvts = partition[:,j, :]\n",
    "            print('Min ' + abt_header[j] + ': ' + str(np.min(mvts)))\n",
    "            print('Max ' + abt_header[j] + ': ' + str(np.max(mvts)))\n",
    "\n",
    "            print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "30b9ce32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Partition: 1\n",
      "Min R_VALUE: 0.0017177569608999\n",
      "Max R_VALUE: 5.674543728986246\n",
      "\n",
      "\n",
      "Min TOTUSJH: 0.0001809902644906\n",
      "Max TOTUSJH: 6078.23759924654\n",
      "\n",
      "\n",
      "Min TOTBSQ: 31324.6457\n",
      "Max TOTBSQ: 102662810417.13412\n",
      "\n",
      "\n",
      "Min TOTPOT: 5.350247853635544e+16\n",
      "Max TOTPOT: 2.820688751045556e+25\n",
      "\n",
      "\n",
      "Min TOTUSJZ: 128034521.39929274\n",
      "Max TOTUSJZ: 187665592928940.44\n",
      "\n",
      "\n",
      "Min ABSNJZH: 5.330749738963192e-06\n",
      "Max ABSNJZH: 2582.482977714385\n",
      "\n",
      "\n",
      "Min SAVNCPP: 5204636.654815674\n",
      "Max SAVNCPP: 92546068787422.56\n",
      "\n",
      "\n",
      "Min USFLUX: 6175406931195526.0\n",
      "Max USFLUX: 8.775930293666179e+22\n",
      "\n",
      "\n",
      "Min TOTFZ: -3.928351131267749e+25\n",
      "Max TOTFZ: 1.28553633794317e+24\n",
      "\n",
      "\n",
      "Min MEANPOT: 40.286658827899146\n",
      "Max MEANPOT: 55455407.40811653\n",
      "\n",
      "\n",
      "Min EPSX: -0.4803478799556892\n",
      "Max EPSX: 0.4596854816593336\n",
      "\n",
      "\n",
      "Min EPSY: -0.4722807843417359\n",
      "Max EPSY: 0.4917124917771694\n",
      "\n",
      "\n",
      "Min EPSZ: -0.9905897598353742\n",
      "Max EPSZ: 0.9992097972611852\n",
      "\n",
      "\n",
      "Min MEANSHR: 2.6141170203785107\n",
      "Max MEANSHR: 78.62631173441771\n",
      "\n",
      "\n",
      "Min SHRGT45: 0.0097399435083276\n",
      "Max SHRGT45: 100.0\n",
      "\n",
      "\n",
      "Min MEANGAM: 8.845779332189387\n",
      "Max MEANGAM: 88.86104778795477\n",
      "\n",
      "\n",
      "Min MEANGBT: 9.804504549235924\n",
      "Max MEANGBT: 265.1900514169275\n",
      "\n",
      "\n",
      "Min MEANGBZ: 10.045626365553636\n",
      "Max MEANGBZ: 267.4541105714504\n",
      "\n",
      "\n",
      "Min MEANGBH: 8.44206991237736\n",
      "Max MEANGBH: 160.69197597766728\n",
      "\n",
      "\n",
      "Min MEANJZH: -0.1756593867129903\n",
      "Max MEANJZH: 0.3629778100906229\n",
      "\n",
      "\n",
      "Min TOTFY: -4.6219334363549833e+24\n",
      "Max TOTFY: 9.433045922509956e+24\n",
      "\n",
      "\n",
      "Min MEANJZD: -23.705855576272512\n",
      "Max MEANJZD: 56.06641502188035\n",
      "\n",
      "\n",
      "Min MEANALP: -26.490511159720175\n",
      "Max MEANALP: 2.83325634722499\n",
      "\n",
      "\n",
      "Min TOTFX: -1.1010777054349613e+25\n",
      "Max TOTFX: 1.2591747557655036e+25\n",
      "\n",
      "\n",
      "Partition: 2\n",
      "Min R_VALUE: 0.0015318156473052\n",
      "Max R_VALUE: 5.7279338965740525\n",
      "\n",
      "\n",
      "Min TOTUSJH: 1.9668876933027247e-05\n",
      "Max TOTUSJH: 8954.798814020878\n",
      "\n",
      "\n",
      "Min TOTBSQ: 10445.9966\n",
      "Max TOTBSQ: 135338577139.514\n",
      "\n",
      "\n",
      "Min TOTPOT: 6663448041529327.0\n",
      "Max TOTPOT: 8.329858530039478e+25\n",
      "\n",
      "\n",
      "Min TOTUSJZ: 579997.9353226197\n",
      "Max TOTUSJZ: 283397440665556.8\n",
      "\n",
      "\n",
      "Min ABSNJZH: 1.9668876933027247e-05\n",
      "Max ABSNJZH: 2494.961664299553\n",
      "\n",
      "\n",
      "Min SAVNCPP: 560841.6979789734\n",
      "Max SAVNCPP: 105331487353070.02\n",
      "\n",
      "\n",
      "Min USFLUX: 8101072102033398.0\n",
      "Max USFLUX: 1.2980669568113693e+23\n",
      "\n",
      "\n",
      "Min TOTFZ: -5.120875623862436e+25\n",
      "Max TOTFZ: 2.820197216067627e+24\n",
      "\n",
      "\n",
      "Min MEANPOT: 5.01748746959543\n",
      "Max MEANPOT: 51931592.92034819\n",
      "\n",
      "\n",
      "Min EPSX: -0.494682932477407\n",
      "Max EPSX: 0.4658935998148371\n",
      "\n",
      "\n",
      "Min EPSY: -0.4932259673212195\n",
      "Max EPSY: 0.4457508527050046\n",
      "\n",
      "\n",
      "Min EPSZ: -0.9658079065670856\n",
      "Max EPSZ: 0.9981207684826644\n",
      "\n",
      "\n",
      "Min MEANSHR: 1.821556596685629\n",
      "Max MEANSHR: 107.82561461548504\n",
      "\n",
      "\n",
      "Min SHRGT45: 0.006249218847644\n",
      "Max SHRGT45: 100.0\n",
      "\n",
      "\n",
      "Min MEANGAM: 10.90819562157241\n",
      "Max MEANGAM: 88.2434285755629\n",
      "\n",
      "\n",
      "Min MEANGBT: 3.643123256848516\n",
      "Max MEANGBT: 310.70583942089075\n",
      "\n",
      "\n",
      "Min MEANGBZ: 8.117696717665645\n",
      "Max MEANGBZ: 282.3029140834363\n",
      "\n",
      "\n",
      "Min MEANGBH: 4.7359559212182765\n",
      "Max MEANGBH: 187.91199764461663\n",
      "\n",
      "\n",
      "Min MEANJZH: -0.2457382606668008\n",
      "Max MEANJZH: 0.4782307698199214\n",
      "\n",
      "\n",
      "Min TOTFY: -1.1982403910962e+25\n",
      "Max TOTFY: 4.545816981130856e+24\n",
      "\n",
      "\n",
      "Min MEANJZD: -52.27885909585711\n",
      "Max MEANJZD: 26.810904318354932\n",
      "\n",
      "\n",
      "Min MEANALP: -11.92200142112318\n",
      "Max MEANALP: 21.19899131527349\n",
      "\n",
      "\n",
      "Min TOTFX: -1.7210259306154412e+25\n",
      "Max TOTFX: 1.5683117364377617e+25\n",
      "\n",
      "\n",
      "Partition: 3\n",
      "Min R_VALUE: 0.0140561891589205\n",
      "Max R_VALUE: 5.838592367672832\n",
      "\n",
      "\n",
      "Min TOTUSJH: 0.0010654980038891\n",
      "Max TOTUSJH: 11523.247939995596\n",
      "\n",
      "\n",
      "Min TOTBSQ: 2640.0174\n",
      "Max TOTBSQ: 143191830656.55402\n",
      "\n",
      "\n",
      "Min TOTPOT: 1.95937478329572e+16\n",
      "Max TOTPOT: 8.684370328288162e+26\n",
      "\n",
      "\n",
      "Min TOTUSJZ: 129484521.16158916\n",
      "Max TOTUSJZ: 163388300272485.88\n",
      "\n",
      "\n",
      "Min ABSNJZH: 3.228332364879627e-05\n",
      "Max ABSNJZH: 2980.2175877520895\n",
      "\n",
      "\n",
      "Min SAVNCPP: 1674554.33052063\n",
      "Max SAVNCPP: 108546206878419.8\n",
      "\n",
      "\n",
      "Min USFLUX: 6082442017494237.0\n",
      "Max USFLUX: 1.192345805582577e+23\n",
      "\n",
      "\n",
      "Min TOTFZ: -4.91763708474166e+25\n",
      "Max TOTFZ: 3.7706671500710514e+24\n",
      "\n",
      "\n",
      "Min MEANPOT: 14.753838651460788\n",
      "Max MEANPOT: 752881040.7516898\n",
      "\n",
      "\n",
      "Min EPSX: -0.4624714282775882\n",
      "Max EPSX: 0.4627714325137246\n",
      "\n",
      "\n",
      "Min EPSY: -0.4635547166050719\n",
      "Max EPSY: 0.4918620620354002\n",
      "\n",
      "\n",
      "Min EPSZ: -0.9630979767484872\n",
      "Max EPSZ: 0.9957506165328588\n",
      "\n",
      "\n",
      "Min MEANSHR: 2.5705500184273498\n",
      "Max MEANSHR: 135.78427681306937\n",
      "\n",
      "\n",
      "Min SHRGT45: 0.0071882974517485\n",
      "Max SHRGT45: 100.0\n",
      "\n",
      "\n",
      "Min MEANGAM: 11.6042595323034\n",
      "Max MEANGAM: 87.44127601289978\n",
      "\n",
      "\n",
      "Min MEANGBT: 17.35023106495296\n",
      "Max MEANGBT: 281.43096119640734\n",
      "\n",
      "\n",
      "Min MEANGBZ: 18.11649736469726\n",
      "Max MEANGBZ: 268.602990301086\n",
      "\n",
      "\n",
      "Min MEANGBH: 8.032226899964801\n",
      "Max MEANGBH: 192.1200332434927\n",
      "\n",
      "\n",
      "Min MEANJZH: -0.2349188595226632\n",
      "Max MEANJZH: 0.1874780433924202\n",
      "\n",
      "\n",
      "Min TOTFY: -5.959080864227922e+24\n",
      "Max TOTFY: 4.089255556045736e+24\n",
      "\n",
      "\n",
      "Min MEANJZD: -36.05613709409323\n",
      "Max MEANJZD: 27.0718551669903\n",
      "\n",
      "\n",
      "Min MEANALP: -50.79508418456538\n",
      "Max MEANALP: 1.072972109084751\n",
      "\n",
      "\n",
      "Min TOTFX: -1.530704309899048e+25\n",
      "Max TOTFX: 1.7301624107828676e+25\n",
      "\n",
      "\n",
      "Partition: 4\n",
      "Min R_VALUE: 0.0023250761093207\n",
      "Max R_VALUE: 5.8692422867886895\n",
      "\n",
      "\n",
      "Min TOTUSJH: 0.0012219893280563\n",
      "Max TOTUSJH: 35104.921076146085\n",
      "\n",
      "\n",
      "Min TOTBSQ: 26546.8209\n",
      "Max TOTBSQ: 254765154646.3088\n",
      "\n",
      "\n",
      "Min TOTPOT: 3881593379446351.0\n",
      "Max TOTPOT: 1.7107575153890706e+26\n",
      "\n",
      "\n",
      "Min TOTUSJZ: 59449779.36712107\n",
      "Max TOTUSJZ: 952801748062894.6\n",
      "\n",
      "\n",
      "Min ABSNJZH: 1.1976039691496965e-05\n",
      "Max ABSNJZH: 2309.9705935688157\n",
      "\n",
      "\n",
      "Min SAVNCPP: 590889.3755187988\n",
      "Max SAVNCPP: 71663746559013.94\n",
      "\n",
      "\n",
      "Min USFLUX: 5.237805250466754e+16\n",
      "Max USFLUX: 2.077737061737166e+23\n",
      "\n",
      "\n",
      "Min TOTFZ: -7.793318965466436e+25\n",
      "Max TOTFZ: 1.0687770056173226e+25\n",
      "\n",
      "\n",
      "Min MEANPOT: 2.9227887872680514\n",
      "Max MEANPOT: 262226204.84213868\n",
      "\n",
      "\n",
      "Min EPSX: -0.4612924166625704\n",
      "Max EPSX: 0.4621891329101379\n",
      "\n",
      "\n",
      "Min EPSY: -0.4280386693942989\n",
      "Max EPSY: 0.4712412111808301\n",
      "\n",
      "\n",
      "Min EPSZ: -0.9642183383977802\n",
      "Max EPSZ: 0.9705046427216184\n",
      "\n",
      "\n",
      "Min MEANSHR: 1.2411197572202466\n",
      "Max MEANSHR: 79.85815209861008\n",
      "\n",
      "\n",
      "Min SHRGT45: 0.0066498204548477\n",
      "Max SHRGT45: 100.0\n",
      "\n",
      "\n",
      "Min MEANGAM: 10.204432101106976\n",
      "Max MEANGAM: 84.39027500060624\n",
      "\n",
      "\n",
      "Min MEANGBT: 11.63608452718232\n",
      "Max MEANGBT: 663.8473089473997\n",
      "\n",
      "\n",
      "Min MEANGBZ: 6.193585391354496\n",
      "Max MEANGBZ: 534.8361166398905\n",
      "\n",
      "\n",
      "Min MEANGBH: 2.882738123180556\n",
      "Max MEANGBH: 439.71895086336855\n",
      "\n",
      "\n",
      "Min MEANJZH: -0.1772692765115549\n",
      "Max MEANJZH: 0.2355185888004908\n",
      "\n",
      "\n",
      "Min TOTFY: -1.7020975301970745e+25\n",
      "Max TOTFY: 4.2824310975425633e+24\n",
      "\n",
      "\n",
      "Min MEANJZD: -44.20589617036227\n",
      "Max MEANJZD: 25.255055468823628\n",
      "\n",
      "\n",
      "Min MEANALP: -1.1149487673691083\n",
      "Max MEANALP: 1.3102523164393127\n",
      "\n",
      "\n",
      "Min TOTFX: -2.533082880887489e+25\n",
      "Max TOTFX: 2.0639339593598213e+25\n",
      "\n",
      "\n",
      "Partition: 5\n",
      "Min R_VALUE: 0.0010349423943981\n",
      "Max R_VALUE: 5.642606630525672\n",
      "\n",
      "\n",
      "Min TOTUSJH: 0.0011812103592159\n",
      "Max TOTUSJH: 7797.952848290409\n",
      "\n",
      "\n",
      "Min TOTBSQ: 24762.25410000001\n",
      "Max TOTBSQ: 91829446280.39365\n",
      "\n",
      "\n",
      "Min TOTPOT: 1208161612427105.8\n",
      "Max TOTPOT: 9.525020373295788e+25\n",
      "\n",
      "\n",
      "Min TOTUSJZ: 64959775.5607992\n",
      "Max TOTUSJZ: 130996191658773.5\n",
      "\n",
      "\n",
      "Min ABSNJZH: 5.3793190807338576e-06\n",
      "Max ABSNJZH: 4084.21083468108\n",
      "\n",
      "\n",
      "Min SAVNCPP: 558045.8722267151\n",
      "Max SAVNCPP: 106807338141026.6\n",
      "\n",
      "\n",
      "Min USFLUX: 1.6680235134396924e+16\n",
      "Max USFLUX: 9.694153244670555e+22\n",
      "\n",
      "\n",
      "Min TOTFZ: -3.950988721658915e+25\n",
      "Max TOTFZ: 3.020963672460711e+23\n",
      "\n",
      "\n",
      "Min MEANPOT: 0.9097295666781978\n",
      "Max MEANPOT: 87303048.33360408\n",
      "\n",
      "\n",
      "Min EPSX: -0.4935181064873128\n",
      "Max EPSX: 0.4692977473221275\n",
      "\n",
      "\n",
      "Min EPSY: -0.4909142114220634\n",
      "Max EPSY: 0.4807367312895781\n",
      "\n",
      "\n",
      "Min EPSZ: -0.9899796152570242\n",
      "Max EPSZ: 0.9872585428319306\n",
      "\n",
      "\n",
      "Min MEANSHR: 0.8556942634296897\n",
      "Max MEANSHR: 87.56729402779474\n",
      "\n",
      "\n",
      "Min SHRGT45: 0.0071164247082265\n",
      "Max SHRGT45: 100.0\n",
      "\n",
      "\n",
      "Min MEANGAM: 10.381843098722891\n",
      "Max MEANGAM: 85.42196199625586\n",
      "\n",
      "\n",
      "Min MEANGBT: 8.675599130846619\n",
      "Max MEANGBT: 269.2402943814664\n",
      "\n",
      "\n",
      "Min MEANGBZ: 7.564178739294836\n",
      "Max MEANGBZ: 271.3352733607459\n",
      "\n",
      "\n",
      "Min MEANGBH: 4.9933477661574255\n",
      "Max MEANGBH: 160.7700401196855\n",
      "\n",
      "\n",
      "Min MEANJZH: -0.3348725254875295\n",
      "Max MEANJZH: 0.1857676578044046\n",
      "\n",
      "\n",
      "Min TOTFY: -6.511168932789288e+24\n",
      "Max TOTFY: 6.557398786286534e+24\n",
      "\n",
      "\n",
      "Min MEANJZD: -38.199748064575\n",
      "Max MEANJZD: 50.54066661100982\n",
      "\n",
      "\n",
      "Min MEANALP: -12.054423298711358\n",
      "Max MEANALP: 2.1391485053269084\n",
      "\n",
      "\n",
      "Min TOTFX: -1.2653822957479718e+25\n",
      "Max TOTFX: 1.2131339558980858e+25\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_values(imputed_data,1,5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63ec6e16",
   "metadata": {},
   "source": [
    "### Saving Final Data Imputed by FPCKNN"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "457b2a91",
   "metadata": {},
   "source": [
    "We chose to name our imputation technique KnnImputation instead of FPCKNN for saving as a file, as it more clearly indicates that it is based on the KNN technique."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5109f09d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/3_KnnImputation/\"\n",
    "imputed_data = []\n",
    "\n",
    "number_of_partitions = 5\n",
    "for i in range(1,number_of_partitions +1):\n",
    "# Load the array with Pickle\n",
    "    with open(data_dir + \"Partition\" + str(i) + \"_KnnImputation\" + \".pkl\", 'rb') as f:\n",
    "        imputed_data.append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0435e333",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "labels = []\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/2_Labels/\"\n",
    "\n",
    "for i in range(1,6):\n",
    "    labels.append(pd.read_csv(data_dir + \"Partition\" + str(i) + \"_labels.csv\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ecb1fab3",
   "metadata": {},
   "source": [
    "## Doing Imputation on Concatenated Multivariate Time Series Data (2D)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef3adbf5",
   "metadata": {},
   "source": [
    "### Converting the 3D Multivariate Time Series Data to a 2D Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a709c208",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concatenation with Shuffle\n",
    "\n",
    "def multi_to_uni(start_partition, end_partition, data_dir, data, labels, name):\n",
    "    category_mapping = {'X': 1, 'M': 1, 'B': 0, 'C': 0, 'FQ': 0}\n",
    "    \n",
    "    num_attributes = 25\n",
    "    num_timestamps = 60\n",
    "    \n",
    "    for i in range(start_partition-1,end_partition):\n",
    "        \n",
    "        the_labels = pd.DataFrame()\n",
    "        the_labels['FLARE_CLASS'] = labels[i]['FLARE_CLASS'].map(category_mapping)\n",
    "        new_partition = np.zeros((np.array(data[i]).shape[2], num_timestamps*(num_attributes-1)))\n",
    "        new_partition_label = np.zeros(new_partition.shape[0])\n",
    "        \n",
    "        each_partition = np.zeros((num_timestamps, num_attributes, new_partition.shape[0]))\n",
    "        each_partition = np.array(data[i])\n",
    "        \n",
    "        with tqdm(new_partition.shape[0]) as pbar:\n",
    "            for j in range(0,new_partition.shape[0]):\n",
    "                new_column = np.zeros((num_timestamps,num_attributes)) \n",
    "                new_column = each_partition[:,:,j]\n",
    "\n",
    "                flettened = np.zeros(num_timestamps*(num_attributes-1))\n",
    "\n",
    "                for m in range(1,num_attributes):\n",
    "                    flettened[(m-1)*num_timestamps:m*num_timestamps] = new_column[:,m]\n",
    "\n",
    "                new_partition[j,:] = flettened\n",
    "                new_partition_label[j] = the_labels.iloc[j]\n",
    "                \n",
    "                \n",
    "                pbar.update(1)\n",
    "        \n",
    "        print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(new_partition).any()))\n",
    "        X_train = new_partition\n",
    "        Y_train = new_partition_label\n",
    "\n",
    "\n",
    "        num_samples = X_train.shape[0]\n",
    "        shuffle_indices = np.random.permutation(num_samples)\n",
    "\n",
    "        X_train_shuffled = X_train[shuffle_indices]\n",
    "        Y_train_shuffled = Y_train[shuffle_indices]\n",
    "    \n",
    "\n",
    "        with open(data_dir + \"Partition\" + str(i+1) \n",
    "                       + \"_Concatenation_\" + name + \".pkl\", 'wb') as f:\n",
    "            pickle.dump(X_train_shuffled, f)\n",
    "\n",
    "        with open(data_dir + \"Partition\" + str(i+1) \n",
    "                       + \"_Labels_Concatenation_\" + name + \".pkl\", 'wb') as f:\n",
    "            pickle.dump(Y_train_shuffled, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "657a835a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "73492it [00:01, 41704.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "88557it [00:02, 37354.93it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P2 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "42510it [00:01, 42142.43it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P3 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "51261it [00:01, 43855.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P4 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "75365it [00:01, 43659.81it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_1_FinalData_Concatenation_KnnImputation/\"\n",
    "\n",
    "multi_to_uni(1, 5, data_dir, imputed_data, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3c18e82",
   "metadata": {},
   "source": [
    "### Mean Imputation Technique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "88a6f835",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/3_2_BaselineImputation/\"\n",
    "imputed_data = []\n",
    "\n",
    "number_of_partitions = 5\n",
    "for i in range(1,number_of_partitions +1):\n",
    "# Load the array with Pickle\n",
    "    with open(data_dir + \"Partition\" + str(i) + \"_MeanImputation\" + \".pkl\", 'rb') as f:\n",
    "        imputed_data.append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3c62a12b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "labels = []\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/2_Labels/\"\n",
    "\n",
    "for i in range(1,6):\n",
    "    labels.append(pd.read_csv(data_dir + \"Partition\" + str(i) + \"_labels.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9c016e98",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "73492it [00:01, 40753.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "88557it [00:02, 37072.17it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P2 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "42510it [00:01, 38876.49it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P3 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "51261it [00:01, 40859.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P4 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "75365it [00:01, 40216.50it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_0_FinalData_BaseLineImputation/\"\n",
    "name = 'MeanImputation'\n",
    "multi_to_uni(1, 5, data_dir, imputed_data, labels, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13eac80b",
   "metadata": {},
   "source": [
    "### Nextvalue Imputation Technique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "476b13e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/3_2_BaselineImputation/\"\n",
    "imputed_data = []\n",
    "\n",
    "number_of_partitions = 5\n",
    "for i in range(1,number_of_partitions +1):\n",
    "# Load the array with Pickle\n",
    "    with open(data_dir + \"Partition\" + str(i) + \"_NextvalueImputation\" + \".pkl\", 'rb') as f:\n",
    "        imputed_data.append(pickle.load(f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5152b152",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "labels = []\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/2_Labels/\"\n",
    "\n",
    "for i in range(1,6):\n",
    "    labels.append(pd.read_csv(data_dir + \"Partition\" + str(i) + \"_labels.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "352c9f93",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "73492it [00:01, 42710.72it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "88557it [00:02, 40533.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P2 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "42510it [00:01, 39358.24it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P3 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "51261it [00:01, 41648.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P4 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "75365it [00:01, 40292.19it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_0_FinalData_BaseLineImputation/\"\n",
    "name = 'NextvalueImputation'\n",
    "multi_to_uni(1, 5, data_dir, imputed_data, labels, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a9e16c7",
   "metadata": {},
   "source": [
    "## Doing Imputation on Vector Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f33a3337",
   "metadata": {},
   "source": [
    "For each time series feature of SWAN-SF, we calculate nine statistical features to convert the multivariate time series data into vector data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "35482ad8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# NewFeatures with Shuffle\n",
    "\n",
    "def new_Features_pkl(start_partition, end_partition, data_dir, data, labels, name):\n",
    "    category_mapping = {'X': 1, 'M': 1, 'B': 0, 'C': 0, 'FQ': 0}\n",
    "\n",
    "    number_of_new_features = 9\n",
    "    num_attributes = 25\n",
    "    num_timestamps = 60\n",
    "    \n",
    "    for i in range(start_partition-1,end_partition):\n",
    "        \n",
    "        the_labels = pd.DataFrame()\n",
    "        the_labels['FLARE_CLASS'] = labels[i]['FLARE_CLASS'].map(category_mapping)\n",
    "        new_partition = np.zeros((np.array(data[i]).shape[2], number_of_new_features*24))\n",
    "        new_partition_label = np.zeros(new_partition.shape[0])\n",
    "        \n",
    "        each_partition = np.zeros((num_timestamps, num_attributes, new_partition.shape[0]))\n",
    "        each_partition = np.array(data[i])\n",
    "        \n",
    "        with tqdm(new_partition.shape[0]) as pbar:\n",
    "            for j in range(0,new_partition.shape[0]):\n",
    "                new_column = np.zeros((num_timestamps,num_attributes)) \n",
    "                new_column = each_partition[:,:,j]\n",
    "                \n",
    "\n",
    "                new_features = np.zeros(number_of_new_features*(num_attributes-1))\n",
    "\n",
    "                for m in range(1,num_attributes):\n",
    "                    \n",
    "                    mean = np.mean(new_column[:,m])\n",
    "                    new_features[((m-1)*number_of_new_features) + 0] = mean\n",
    "                    median = np.median(new_column[:,m])\n",
    "                    new_features[((m-1)*number_of_new_features) + 1] = median\n",
    "                    std = np.std(new_column[:,m])\n",
    "                    new_features[((m-1)*number_of_new_features) + 2] = std\n",
    "                    \n",
    "                    skewness = skew(new_column[:,m])\n",
    "                    if np.isnan(skewness):\n",
    "                        skewness = new_partition[j-1, ((m-1)*number_of_new_features) + 3]\n",
    "                    new_features[((m-1)*number_of_new_features) + 3] = skewness\n",
    "                    \n",
    "                    kurtosis_value = kurtosis(new_column[:,m])\n",
    "                    if np.isnan(kurtosis_value):\n",
    "                        kurtosis_value = new_partition[j-1, ((m-1)*number_of_new_features) + 4]\n",
    "                    new_features[((m-1)*number_of_new_features) + 4] = kurtosis_value\n",
    "                    \n",
    "                    indices = np.arange(num_timestamps)\n",
    "                    weight_array = indices / num_timestamps\n",
    "                    weighted_avg = np.average(new_column[:,m], weights=weight_array)\n",
    "                    if np.isnan(weighted_avg):\n",
    "                        weighted_avg = new_partition[j-1, ((m-1)*number_of_new_features) + 5]\n",
    "                    new_features[((m-1)*number_of_new_features) + 5] = weighted_avg\n",
    "                    \n",
    "                    last_value = new_column[59,m]\n",
    "                    new_features[((m-1)*number_of_new_features) + 6] = last_value\n",
    "                    first_value = new_column[0,m]\n",
    "                    new_features[((m-1)*number_of_new_features) + 7] = first_value\n",
    "                    \n",
    "                    numerator = np.sum((new_column[:,m] - mean) * (indices - np.mean(indices)))\n",
    "                    denominator = np.sum((new_column[:,m] - mean) ** 2)\n",
    "                    slope = numerator / denominator\n",
    "                    if np.isnan(slope):\n",
    "                        slope = new_partition[j-1, ((m-1)*number_of_new_features) + 8]\n",
    "                    new_features[((m-1)*number_of_new_features) + 8] = slope\n",
    "                    \n",
    "                    if all(value == 1.0 for value in new_column[:,m]):\n",
    "                        new_features[((m-1)*number_of_new_features) + 0 : ((m-1)*number_of_new_features) + 9] = np.ones(9)\n",
    "                \n",
    "                \n",
    "                if np.isnan(new_features).any():\n",
    "                    print('nan')\n",
    "                    print()\n",
    "                    mean_non_zero = np.mean(new_features[~np.isnan(new_features)])\n",
    "                    new_features[np.isnan(new_features)] = mean_non_zero    \n",
    "                    \n",
    "                new_partition[j,:] = new_features\n",
    "                new_partition_label[j] = the_labels.iloc[j]\n",
    "                \n",
    "                pbar.update(1)\n",
    "                \n",
    "\n",
    "        print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(new_partition).any()))  \n",
    "        X_train = new_partition\n",
    "        Y_train = new_partition_label\n",
    "\n",
    "\n",
    "        num_samples = X_train.shape[0]\n",
    "        shuffle_indices = np.random.permutation(num_samples)\n",
    "\n",
    "        X_train_shuffled = X_train[shuffle_indices]\n",
    "        Y_train_shuffled = Y_train[shuffle_indices]\n",
    "    \n",
    "\n",
    "        with open(data_dir + \"Partition\" + str(i+1) \n",
    "                       + \"_NewFeatures_\" + name + \".pkl\", 'wb') as f:\n",
    "            pickle.dump(X_train_shuffled, f)\n",
    "\n",
    "        with open(data_dir + \"Partition\" + str(i+1) \n",
    "                       + \"_Labels_NewFeatures_\" + name + \".pkl\", 'wb') as f:\n",
    "            pickle.dump(Y_train_shuffled, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c502ffb9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "from scipy.stats import skew, kurtosis\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_2_FinalData_NewFeatures_KnnImputation/\"\n",
    "\n",
    "new_Features_pkl(1, 5, data_dir, imputed_data, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd88822a",
   "metadata": {},
   "source": [
    "### Mean Imputation Technique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b774675e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1897it [00:11, 157.64it/s]/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:37: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  skewness = skew(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:42: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  kurtosis_value = kurtosis(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:61: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  slope = numerator / denominator\n",
      "73492it [07:41, 159.09it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1084it [00:06, 159.20it/s]/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:37: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  skewness = skew(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:42: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  kurtosis_value = kurtosis(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:61: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  slope = numerator / denominator\n",
      "88557it [09:21, 157.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P2 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3381it [00:21, 161.33it/s]/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:37: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  skewness = skew(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:42: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  kurtosis_value = kurtosis(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:61: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  slope = numerator / denominator\n",
      "42510it [04:28, 158.60it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P3 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3733it [00:23, 159.13it/s]/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:37: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  skewness = skew(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:42: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  kurtosis_value = kurtosis(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:61: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  slope = numerator / denominator\n",
      "51261it [05:24, 157.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P4 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3150it [00:19, 160.17it/s]/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:37: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  skewness = skew(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:42: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  kurtosis_value = kurtosis(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_6471/1406415825.py:61: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  slope = numerator / denominator\n",
      "75365it [07:53, 159.07it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "from scipy.stats import skew, kurtosis\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_0_FinalData_BaseLineImputation/\"\n",
    "name = 'MeanImputation'\n",
    "new_Features_pkl(1, 5, data_dir, imputed_data, labels, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fa3d6e10",
   "metadata": {},
   "source": [
    "### Nextvalue Imputation Technique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4ab585e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1887it [00:13, 139.05it/s]/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:37: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  skewness = skew(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:42: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  kurtosis_value = kurtosis(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:61: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  slope = numerator / denominator\n",
      "73492it [10:57, 111.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1078it [00:10, 111.47it/s]/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:37: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  skewness = skew(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:42: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  kurtosis_value = kurtosis(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:61: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  slope = numerator / denominator\n",
      "88557it [14:19, 103.08it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P2 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3383it [00:35, 107.81it/s]/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:37: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  skewness = skew(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:42: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  kurtosis_value = kurtosis(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:61: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  slope = numerator / denominator\n",
      "42510it [07:08, 99.31it/s] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P3 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3743it [00:33, 105.28it/s]/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:37: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  skewness = skew(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:42: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  kurtosis_value = kurtosis(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:61: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  slope = numerator / denominator\n",
      "51261it [08:08, 104.83it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P4 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3156it [00:27, 103.90it/s]/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:37: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  skewness = skew(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:42: RuntimeWarning: Precision loss occurred in moment calculation due to catastrophic cancellation. This occurs when the data are nearly identical. Results may be unreliable.\n",
      "  kurtosis_value = kurtosis(new_column[:,m])\n",
      "/var/folders/fx/gjhbmrbj5jn295_9wrqpbsv80000gn/T/ipykernel_19762/1406415825.py:61: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  slope = numerator / denominator\n",
      "75365it [11:33, 108.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "from scipy.stats import skew, kurtosis\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "scaler = MinMaxScaler()\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_0_FinalData_BaseLineImputation/\"\n",
    "name = 'NextvalueImputation'\n",
    "new_Features_pkl(1, 5, data_dir, imputed_data, labels, name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d49f5fc7",
   "metadata": {},
   "source": [
    "## Final Data Without Imputation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e7a58a7",
   "metadata": {},
   "source": [
    "### Concatenated Multivariate Time Series Data (2D)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "37db2e75",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Concatenation with Shuffle\n",
    "\n",
    "def multi_to_uni(start_partition, end_partition, data_dir, data, labels):\n",
    "    category_mapping = {'X': 1, 'M': 1, 'B': 0, 'C': 0, 'FQ': 0}\n",
    "    \n",
    "    num_attributes = 25\n",
    "    num_timestamps = 60\n",
    "    \n",
    "    for i in range(start_partition-1,end_partition):\n",
    "        \n",
    "        the_labels = pd.DataFrame()\n",
    "        the_labels['FLARE_CLASS'] = labels[i]['FLARE_CLASS'].map(category_mapping)\n",
    "        new_partition = np.zeros((np.array(data[i]).shape[2], num_timestamps*(num_attributes-1)))\n",
    "        new_partition_label = np.zeros(new_partition.shape[0])\n",
    "        \n",
    "        each_partition = np.zeros((num_timestamps, num_attributes, new_partition.shape[0]))\n",
    "        each_partition = np.array(data[i])\n",
    "        \n",
    "        with tqdm(new_partition.shape[0]) as pbar:\n",
    "            for j in range(0,new_partition.shape[0]):\n",
    "                new_column = np.zeros((num_timestamps,num_attributes)) \n",
    "                new_column = each_partition[:,:,j]\n",
    "                \n",
    "                new_column = np.nan_to_num(new_column, nan=0.0)\n",
    "                \n",
    "                flettened = np.zeros(num_timestamps*(num_attributes-1))\n",
    "\n",
    "                for m in range(1,num_attributes):\n",
    "                    flettened[(m-1)*num_timestamps:m*num_timestamps] = new_column[:,m]\n",
    "\n",
    "                new_partition[j,:] = flettened\n",
    "                new_partition_label[j] = the_labels.iloc[j]\n",
    "                \n",
    "                \n",
    "                pbar.update(1)\n",
    "        \n",
    "        print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(new_partition).any()))\n",
    "        X_train = new_partition\n",
    "        Y_train = new_partition_label\n",
    "\n",
    "\n",
    "        num_samples = X_train.shape[0]\n",
    "        shuffle_indices = np.random.permutation(num_samples)\n",
    "\n",
    "        X_train_shuffled = X_train[shuffle_indices]\n",
    "        Y_train_shuffled = Y_train[shuffle_indices]\n",
    "    \n",
    "\n",
    "        with open(data_dir + \"Partition\" + str(i+1) \n",
    "                       + \"_Concatenation_Raw\" + \".pkl\", 'wb') as f:\n",
    "            pickle.dump(X_train_shuffled, f)\n",
    "\n",
    "        with open(data_dir + \"Partition\" + str(i+1) \n",
    "                       + \"_Labels_Concatenation_Raw\" + \".pkl\", 'wb') as f:\n",
    "            pickle.dump(Y_train_shuffled, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "03152a53",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "73492it [00:02, 28942.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "88557it [00:03, 28486.30it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P2 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "42510it [00:01, 27675.02it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P3 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "51261it [00:01, 28415.85it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P4 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "75365it [00:02, 26911.71it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_6_FinalData_Concatenation_Raw/\"\n",
    "\n",
    "multi_to_uni(1, 5, data_dir, raw_data, labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c9cb867",
   "metadata": {},
   "source": [
    "### Vector Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9f4c85c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# NewFeatures with Shuffle\n",
    "\n",
    "def new_Features_pkl(start_partition, end_partition, data_dir, data, labels):\n",
    "    category_mapping = {'X': 1, 'M': 1, 'B': 0, 'C': 0, 'FQ': 0}\n",
    "\n",
    "    number_of_new_features = 9\n",
    "    num_attributes = 25\n",
    "    num_timestamps = 60\n",
    "    \n",
    "    for i in range(start_partition-1,end_partition):\n",
    "        \n",
    "        the_labels = pd.DataFrame()\n",
    "        the_labels['FLARE_CLASS'] = labels[i]['FLARE_CLASS'].map(category_mapping)\n",
    "        new_partition = np.zeros((np.array(data[i]).shape[2], number_of_new_features*24))\n",
    "        new_partition_label = np.zeros(new_partition.shape[0])\n",
    "        \n",
    "        each_partition = np.zeros((num_timestamps, num_attributes, new_partition.shape[0]))\n",
    "        each_partition = np.array(data[i])\n",
    "        \n",
    "        with tqdm(new_partition.shape[0]) as pbar:\n",
    "            for j in range(0,new_partition.shape[0]):\n",
    "                new_column = np.zeros((num_timestamps,num_attributes)) \n",
    "                new_column = each_partition[:,:,j]\n",
    "                \n",
    "                new_column = np.nan_to_num(new_column, nan=0.0)\n",
    "\n",
    "                new_features = np.zeros(number_of_new_features*(num_attributes-1))\n",
    "\n",
    "                for m in range(1,num_attributes):\n",
    "                    all_zeros = np.all(new_column[:,m] == 0.0)\n",
    "                    if all_zeros:\n",
    "                        new_features[((m-1)*number_of_new_features):((m-1)*number_of_new_features)+9] = 0\n",
    "                    else:\n",
    "                        mean = np.mean(new_column[:,m])\n",
    "                        new_features[((m-1)*number_of_new_features) + 0] = mean\n",
    "                        median = np.median(new_column[:,m])\n",
    "                        new_features[((m-1)*number_of_new_features) + 1] = median\n",
    "                        std = np.std(new_column[:,m])\n",
    "                        new_features[((m-1)*number_of_new_features) + 2] = std\n",
    "\n",
    "                        skewness = skew(new_column[:,m])\n",
    "                        if np.isreal(skewness) == False:\n",
    "                            skewness = new_partition[j-1, ((m-1)*number_of_new_features) + 3]\n",
    "                        new_features[((m-1)*number_of_new_features) + 3] = skewness\n",
    "\n",
    "                        kurtosis_value = kurtosis(new_column[:,m])\n",
    "                        if np.isreal(kurtosis_value) == False:\n",
    "                            kurtosis_value = new_partition[j-1, ((m-1)*number_of_new_features) + 4]\n",
    "                        new_features[((m-1)*number_of_new_features) + 4] = kurtosis_value\n",
    "\n",
    "                        indices = np.arange(num_timestamps)\n",
    "                        weight_array = indices / num_timestamps\n",
    "                        weighted_avg = np.average(new_column[:,m], weights=weight_array)\n",
    "                        if weighted_avg == np.nan:\n",
    "                            weighted_avg = new_partition[j-1, ((m-1)*number_of_new_features) + 5]\n",
    "                        new_features[((m-1)*number_of_new_features) + 5] = weighted_avg\n",
    "\n",
    "                        last_value = new_column[59,m]\n",
    "                        new_features[((m-1)*number_of_new_features) + 6] = last_value\n",
    "                        first_value = new_column[0,m]\n",
    "                        new_features[((m-1)*number_of_new_features) + 7] = first_value\n",
    "\n",
    "                        numerator = np.sum((new_column[:,m] - mean) * (indices - np.mean(indices)))\n",
    "                        denominator = np.sum((new_column[:,m] - mean) ** 2)\n",
    "                        slope = numerator / denominator\n",
    "                        if np.isreal(slope) == False:\n",
    "                            slope = new_partition[j-1, ((m-1)*number_of_new_features) + 8]\n",
    "                        new_features[((m-1)*number_of_new_features) + 8] = slope\n",
    "                \n",
    "                new_partition[j,:] = new_features\n",
    "                new_partition_label[j] = the_labels.iloc[j]\n",
    "                \n",
    "                pbar.update(1)\n",
    "                \n",
    "\n",
    "        print(\"P\"+str(i+1)+\" Nan-Value: \"+ str(np.isnan(new_partition).any()))  \n",
    "        X_train = new_partition\n",
    "        Y_train = new_partition_label\n",
    "\n",
    "\n",
    "        num_samples = X_train.shape[0]\n",
    "        shuffle_indices = np.random.permutation(num_samples)\n",
    "\n",
    "        X_train_shuffled = X_train[shuffle_indices]\n",
    "        Y_train_shuffled = Y_train[shuffle_indices]\n",
    "    \n",
    "\n",
    "        with open(data_dir + \"Partition\" + str(i+1) \n",
    "                       + \"_NewFeatures_Raw\" + \".pkl\", 'wb') as f:\n",
    "            pickle.dump(X_train_shuffled, f)\n",
    "\n",
    "        with open(data_dir + \"Partition\" + str(i+1) \n",
    "                       + \"_Labels_NewFeatures_Raw\" + \".pkl\", 'wb') as f:\n",
    "            pickle.dump(Y_train_shuffled, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "da8400bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "73492it [07:42, 158.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P1 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "88557it [09:06, 162.00it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P2 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "42510it [04:25, 160.27it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P3 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "51261it [05:19, 160.53it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P4 Nan-Value: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "75365it [07:49, 160.61it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P5 Nan-Value: False\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "from scipy.stats import skew, kurtosis\n",
    "\n",
    "data_dir = \"/Users/samskanderi/Documents/Research_Project/SWANSF/code/4_7_FinalData_NewFeatures_Raw/\"\n",
    "\n",
    "new_Features_pkl(1, 5, data_dir, raw_data, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "244eb710",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}