SHL_import_featureExtraction.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SHL_import_featureExtraction.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNuDPpbqf9nmRS7DUaEHZT+",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/farhanfuadabir/SHL2020/blob/master/SHL_import_featureExtraction.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Gzblo40DBOZ",
        "outputId": "ec2e33c7-d1d5-4ed1-94ef-f3f186651020",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m8qSarrtP2gI"
      },
      "source": [
        "# Dependencies\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import pickle as pk\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
        "\n",
        "\n",
        "# Functions\n",
        "\n",
        "def magnitude(x, y, z):\n",
        "  return np.sqrt(x**2 + y**2 + z**2)\n",
        "\n",
        "\n",
        "def entrop(pk, axis=0):\n",
        "\n",
        "  from scipy.special import entr\n",
        "\n",
        "  sumPk = np.sum(pk, axis=axis, keepdims=True)\n",
        "  sumPk = np.where(sumPk == 0, 0.000001, sumPk)\n",
        "  pk = pk / sumPk\n",
        "  vec = entr(pk)\n",
        "  S = np.sum(vec, axis=axis)\n",
        "  return S\n",
        "\n",
        "def autocorrelation(x, axis=0):\n",
        "\n",
        "  import numpy as np\n",
        "  \n",
        "  def autocorr_along_axis(x):\n",
        "    result = np.correlate(x, x, mode='full')\n",
        "    return result[result.size // 2:]\n",
        "  return np.apply_along_axis(autocorr_along_axis, axis, x)\n",
        "\n",
        "\n",
        "def zero_crossing_rate(X, axis=0):\n",
        "\n",
        "  import numpy as np\n",
        "  X = np.nan_to_num(X)\n",
        "\n",
        "  def zcr_along_axis(X):\n",
        "    temp = np.where(X>=0, 1, -1)\n",
        "    for i in range (0, temp.shape[0]):    \n",
        "      if i == 0:\n",
        "        prev_i = i\n",
        "        S = 0\n",
        "        continue\n",
        "      S += abs(temp[i] - temp[prev_i])\n",
        "      prev_i = i\n",
        "    ans = 0.5*S/X.shape[0]\n",
        "    \n",
        "    return ans\n",
        "\n",
        "  return np.apply_along_axis(zcr_along_axis, axis, X)\n",
        "\n",
        "\n",
        "def first_zero_crossing(x, axis=0):\n",
        "  \n",
        "  import numpy as np\n",
        "  \n",
        "  def fzc_along_axis(k):    \n",
        "    index = np.where(np.diff(np.sign(k)))[0] + 1\n",
        "    if index.size == 0:\n",
        "      return 0\n",
        "    else:\n",
        "      return index[0] \n",
        "      \n",
        "  return np.apply_along_axis(fzc_along_axis, axis, x)\n",
        "\n",
        "\n",
        "\n",
        "def calculateAngle(lx, ly, lz, position='', picklePrefix='', unpicklePath=''):\n",
        "\n",
        "  import numpy as np\n",
        "\n",
        "  gx, gy, gz = process_channels(position, ['Gra_x','Gra_y','Gra_z'], \n",
        "                                      picklePrefix=picklePrefix, \n",
        "                                      unpicklePath=unpicklePath)  \n",
        "  gx = np.nan_to_num(gx)\n",
        "  gy = np.nan_to_num(gy)\n",
        "  gz = np.nan_to_num(gz)\n",
        "  lx = np.nan_to_num(lx)\n",
        "  ly = np.nan_to_num(ly)\n",
        "  lz = np.nan_to_num(lz)\n",
        "  \n",
        "  n = gx*lx + gy*ly + gz*lz\n",
        "  d = magnitude(gx, gy, gz) * magnitude(lx, ly, lz)\n",
        "\n",
        "  # Division by zero is not allowed\n",
        "  # Replacing all 0s with 0.000001\n",
        "  d = np.where(d == 0, 0.000001, d)\n",
        "\n",
        "  theta = np.arccos(n/d)\n",
        "  \n",
        "  return theta\n",
        "\n",
        "\n",
        "def calculate_stat_features(magnitude, prefix=''):\n",
        "  \n",
        "  from scipy.stats import median_absolute_deviation, iqr\n",
        "  import numpy as np\n",
        "  import pandas as pd\n",
        "\n",
        "  mean = np.mean(magnitude, axis=1)\n",
        "  std = np.std(magnitude, axis=1)\n",
        "  max = np.max(magnitude, axis=1)\n",
        "  min = np.min(magnitude, axis=1)\n",
        "  mad = median_absolute_deviation(np.nan_to_num(magnitude), axis=1, nan_policy='omit')\n",
        "  iqr = iqr(magnitude, axis=1, nan_policy='omit')\n",
        "  \n",
        "  corr = autocorrelation(magnitude, axis=1)\n",
        "\n",
        "  maxcorr = np.max(corr, axis=1)\n",
        "  argmax_corr = np.argmax(corr, axis=1)\n",
        "  zcr = zero_crossing_rate(corr, axis=1)\n",
        "  fzc = first_zero_crossing(corr, axis=1)\n",
        "  \n",
        "  columnName = [prefix+'_mean', prefix+'_std', prefix+'_max', prefix+'_min', \n",
        "                prefix+'_mad', prefix+'_iqr', prefix+'_max.corr', \n",
        "                prefix+'_idx.max.corr', prefix+'_zcr', prefix+'_fzc']\n",
        "\n",
        "  stat_feat_df = pd.DataFrame(np.stack((mean, std, max, min, mad, iqr, maxcorr, \n",
        "                                        argmax_corr, zcr, fzc), axis=1), \n",
        "                              columns=columnName)\n",
        "  \n",
        "  return stat_feat_df\n",
        "\n",
        "def calculate_spectral_features(magnitude, fs=100, prefix=''):\n",
        "  \n",
        "  import pandas as pd\n",
        "  import numpy as np\n",
        "  from scipy.stats import kurtosis, skew\n",
        "  from scipy import signal, stats\n",
        "\n",
        "  FREQ,PSD = signal.welch(magnitude, fs, nperseg=magnitude.shape[1], axis=1)\n",
        "  # Max PSD value\n",
        "  maxPSD = np.max(PSD, axis=1)\n",
        "  # Frequency Entropy\n",
        "  entropy = entrop(PSD, axis=1)\n",
        "  # Frequency Center\n",
        "  sumPSD = np.sum(PSD, axis=1)\n",
        "  sumPSD = np.where(sumPSD == 0, 0.000001, sumPSD)\n",
        "  fc = np.sum((FREQ*PSD), axis=1) / sumPSD\n",
        "  # Kurtosis\n",
        "  kurt = kurtosis(PSD, axis=1, nan_policy='omit')\n",
        "  # Skewness\n",
        "  skew = skew(PSD, axis=1, nan_policy='omit')\n",
        "\n",
        "  columnName = [prefix+'_max.psd', prefix+'_entropy', prefix+'_fc', \n",
        "                prefix+'_kurt', prefix+'_skew']\n",
        "\n",
        "  spectral_feat_df = pd.DataFrame(np.stack((maxPSD, entropy, fc, kurt, skew), \n",
        "                                           axis=1), columns=columnName)\n",
        "  \n",
        "  return spectral_feat_df\n",
        "\n",
        "\n",
        "\n",
        "def rotate_axis(ax, ay, az, position='', picklePrefix='', unpicklePath=''):\n",
        "  \n",
        "  import numpy as np\n",
        "\n",
        "  w, x, y, z = process_channels(position, ['Ori_w','Ori_x','Ori_y','Ori_z'], \n",
        "                                picklePrefix=picklePrefix, \n",
        "                                unpicklePath=unpicklePath)\n",
        "\n",
        "  if ax.shape == ay.shape == az.shape == w.shape == x.shape == y.shape == z.shape:\n",
        "    num_column = ax.shape[1]\n",
        "  else:\n",
        "    print(\"Dimention mismatch. Derotation Failed.\")\n",
        "\n",
        "  w = w.ravel()\n",
        "  x = x.ravel()\n",
        "  y = y.ravel()\n",
        "  z = z.ravel()\n",
        "\n",
        "  ax = ax.ravel()\n",
        "  ay = ay.ravel()\n",
        "  az = az.ravel()\n",
        "\n",
        "  Ax = 1 - 2*(y**2 + z**2) * ax + 2*(x*y - w*z) * ay + 2*(x*z + w*y) * az\n",
        "  Ay = 2*(x*y + w*z) * ax + 1 - 2*(x**2 + z**2) * ay + 2*(y*z - w*x) * az\n",
        "  Az = 2*(x*z - w*y) * ax + 2*(y*z + w*x) * ay + 1 - 2*(x**2 + y**2) * az\n",
        "\n",
        "  Ax = Ax.reshape(-1,num_column)\n",
        "  Ay = Ay.reshape(-1,num_column)\n",
        "  Az = Az.reshape(-1,num_column)\n",
        "\n",
        "  return Ax,Ay,Az\n",
        "\n",
        "def process_channels(position, channels, picklePrefix='', unpicklePath='',\n",
        "                     statFeatures=False, spectralFeatures=False, process=None,\n",
        "                     processEachAxis=False, calculateComponent=None, \n",
        "                     outMagnitude=False, isLabel=False, calculateJerk=False, \n",
        "                     prefix=''): \n",
        "  \n",
        "  import pandas as pd\n",
        "  import numpy as np\n",
        "  import pickle as pk\n",
        "  from scipy.stats import mode\n",
        "  \n",
        "  num_channels = len(channels) \n",
        "\n",
        "  print(\"\\nPosition: \" + position)\n",
        "  \n",
        "  # Unpickle the channel[1]\n",
        "  print(\"Unpickling From: \" + unpicklePath + picklePrefix + \"_\" + position.lower() + \"_\" + \n",
        "        channels[0].lower() + \".pickle ...\", end=\" \") \n",
        "  with open(unpicklePath + picklePrefix + \"_\" + position.lower() + \"_\" + channels[0].lower() +\n",
        "            \".pickle\",'rb') as f:\n",
        "    x = pk.load(f)\n",
        "  print('Done')\n",
        "\n",
        "  if num_channels >= 3:\n",
        "    # Unpickle channel[2]\n",
        "    print(\"Unpickling From: \" + unpicklePath + picklePrefix + \"_\" + position.lower() + \"_\" + \n",
        "          channels[1].lower() + \".pickle ...\", end=\" \") \n",
        "    with open(unpicklePath + picklePrefix + \"_\" + position.lower() + \"_\" + channels[1].lower() + \n",
        "              \".pickle\",'rb') as f:\n",
        "      y = pk.load(f)\n",
        "    print('Done')\n",
        "\n",
        "    # Unpickle channel[3]\n",
        "    print(\"Unpickling From: \" + unpicklePath + picklePrefix + \"_\" + position.lower() + \"_\" + \n",
        "          channels[2].lower() + \".pickle ...\", end=\" \") \n",
        "    with open(unpicklePath + picklePrefix + \"_\" + position.lower() + \"_\" + channels[2].lower() + \n",
        "              \".pickle\",'rb') as f:\n",
        "      z = pk.load(f)\n",
        "    print('Done')\n",
        "\n",
        "    if num_channels == 4:\n",
        "      # Unpickle channel[4]\n",
        "      print(\"Unpickling From: \" + unpicklePath + picklePrefix + \"_\" + position.lower() + \"_\" + \n",
        "            channels[3].lower() + \".pickle ...\", end=\" \") \n",
        "      with open(unpicklePath + picklePrefix + \"_\" + position.lower() + \"_\" + channels[3].lower() + \n",
        "                \".pickle\",'rb') as f:\n",
        "        w = pk.load(f)\n",
        "      print('Done')\n",
        "\n",
        "  print(\"Processing... \",end=\" \")\n",
        "  \n",
        "  if num_channels == 4:\n",
        "    # According to the equation the data should be read as: w,x,y,z\n",
        "    print(\"Quarternions Returned\")\n",
        "    return x, y, z, w\n",
        "  elif (num_channels == 1) and (isLabel==True):\n",
        "      label,_ = mode(x,axis=1,nan_policy='omit')\n",
        "      label = label.flatten()\n",
        "      print(\"Labels Returned\")\n",
        "      label = pd.DataFrame(label, columns=['label'])\n",
        "      return label                              # datatype: dataframe\n",
        "  else:\n",
        "    if num_channels == 1:\n",
        "      sq_mag = x\n",
        "    elif num_channels == 3:\n",
        "      if process=='rotate':\n",
        "        print(\"Calculating Axis Rotation...\", end=\" \")\n",
        "        x, y, z = rotate_axis(x, y, z, position=position, \n",
        "                              picklePrefix=picklePrefix, \n",
        "                              unpicklePath=unpicklePath)\n",
        "        sq_mag = magnitude(x, y, z)\n",
        "      elif process is not None:\n",
        "        theta = calculateAngle(x, y, z, position=position, \n",
        "                              picklePrefix=picklePrefix, \n",
        "                              unpicklePath=unpicklePath)\n",
        "        if process=='horizontal':\n",
        "          print(\"Calculating Horizontal Component...\", end=\" \")\n",
        "          sq_mag = magnitude(x, y, z)*np.cos(theta)\n",
        "          if calculateJerk==True:\n",
        "            print(\"Calculating Jerk...\", end=\" \")\n",
        "            sq_mag = np.diff(sq_mag,axis=1)\n",
        "        elif process=='vertical':\n",
        "          print(\"Calculating Vertical Component...\", end=\" \")\n",
        "          sq_mag = magnitude(x, y, z)*np.sin(theta)\n",
        "          if calculateJerk==True:\n",
        "            print(\"Calculating Jerk...\", end=\" \")\n",
        "            sq_mag = np.diff(sq_mag,axis=1)\n",
        "      else:\n",
        "          sq_mag = magnitude(x, y, z)\n",
        "\n",
        "    if (statFeatures==False) and (spectralFeatures==False):\n",
        "      if(outMagnitude==True):\n",
        "        print(\"Magnitude Returned\")\n",
        "        return sq_mag                           # datatype: ndarray\n",
        "      else:\n",
        "        print(\"Triaxial Value Returned\")\n",
        "        return x, y, z                          # datatype: ndarray\n",
        "    else:\n",
        "      if statFeatures==True:\n",
        "        stat_features_mag = calculate_stat_features(sq_mag, prefix=prefix+\"M\")\n",
        "        if (processEachAxis==True) and (num_channels>1):\n",
        "          stat_features_x = calculate_stat_features(x, prefix=prefix+\"X\")\n",
        "          stat_features_y = calculate_stat_features(y, prefix=prefix+\"Y\")\n",
        "          stat_features_z = calculate_stat_features(z, prefix=prefix+\"Z\")\n",
        "\n",
        "          stat_features = pd.concat([stat_features_mag, stat_features_x, \n",
        "                                     stat_features_y, stat_features_z],axis=1)\n",
        "\n",
        "          #stat_features = np.concatenate([stat_features_mag, stat_features_x, \n",
        "          #                                stat_features_y, stat_features_z], \n",
        "          #                               axis=1)\n",
        "        else:\n",
        "          stat_features = stat_features_mag\n",
        "        \n",
        "        if spectralFeatures==False:\n",
        "          print(\"Statistical Features Returned\")\n",
        "          return stat_features                  # datatype: dataframe\n",
        "      if spectralFeatures==True:\n",
        "        spectral_features_mag = calculate_spectral_features(sq_mag,\n",
        "                                                            prefix=prefix+\"M\")\n",
        "        if (processEachAxis==True) and (num_channels>1):\n",
        "          spectral_features_x = calculate_spectral_features(x, prefix=prefix+\"X\")\n",
        "          spectral_features_y = calculate_spectral_features(y, prefix=prefix+\"Y\")\n",
        "          spectral_features_z = calculate_spectral_features(z, prefix=prefix+\"Z\")\n",
        "\n",
        "          spectral_features = pd.concat([spectral_features_mag, \n",
        "                                         spectral_features_x, \n",
        "                                         spectral_features_y, \n",
        "                                         spectral_features_z],axis=1)\n",
        "\n",
        "          #spectral_features = np.concatenate([spectral_features_mag, \n",
        "          #                                    spectral_features_x, \n",
        "          #                                    spectral_features_y, \n",
        "          #                                    spectral_features_z],\n",
        "          #                                   axis=1)\n",
        "        else:\n",
        "          spectral_features = spectral_features_mag\n",
        "\n",
        "        if statFeatures==False:\n",
        "          print(\"Spectral Features Returned\")\n",
        "          return spectral_features              # datatype: dataframe\n",
        "      print(\"Statistical and Spectral Features Returned\")\n",
        "                                                # datatype: dataframe\n",
        "      return pd.concat([stat_features, spectral_features],axis=1)\n",
        "      #return np.concatenate([stat_features,spectral_features],axis=1)\n",
        "    \n",
        "\n",
        "def import_and_pickle(location, positions, filenames, picklePrefix='', \n",
        "                      picklePath=''):\n",
        "  print(\"Importing data from: \" + location + \"...\")\n",
        "  for j in positions:\n",
        "    print(\"\\nPosition: \" + j)\n",
        "    for i in filenames:\n",
        "      # Import from text file\n",
        "      print(\"Importing \" + i + \"...\", end=\" \")\n",
        "      value = np.loadtxt(location + j + \"/\" + i + \".txt\")\n",
        "      print(\"Done | Shape: \", value.shape)\n",
        "\n",
        "      # Check for nan values\n",
        "      nan_count = np.isnan(value).sum()\n",
        "      if nan_count != 0:\n",
        "        print(\"\\tnan detected | : \", nan_count, \" | Filename: \" + i)\n",
        "\n",
        "      # Pickle the values\n",
        "      import pickle as pk\n",
        "      with open(picklePath + picklePrefix + \"_\" + j.lower() + \"_\" + i.lower() + \n",
        "                \".pickle\",'wb') as f:\n",
        "        pk.dump(value,f)\n",
        "      print(\"\\tPickled at: \" + picklePath + picklePrefix + \"_\" + j.lower() + \n",
        "            \"_\" + i.lower() + \".pickle\")\n",
        "      \n",
        "\n",
        "def extract_acc_pickle(positions, picklePrefix='', unpicklePath='', \n",
        "                       picklePath='', prefix='acc'):\n",
        "  # Extract ACC\n",
        "  print(\"Extracting ACC... \", end=\" \")\n",
        "  acc = process_channels(positions, ['Acc_x','Acc_y','Acc_z'], \n",
        "                         picklePrefix=picklePrefix, processEachAxis=True, \n",
        "                         statFeatures=True, spectralFeatures=True, \n",
        "                         process='rotate', unpicklePath=unpicklePath, \n",
        "                         prefix=prefix)\n",
        "  print(\"acc Shape: \", acc.shape)\n",
        "\n",
        "  # Pickle \n",
        "  import pickle as pk\n",
        "  with open(picklePath + picklePrefix + \"_\" + positions.lower() + \"_ACC.pickle\",\n",
        "            'wb') as f:\n",
        "    pk.dump(acc,f)\n",
        "  print(\"\\nPickled at: \" + picklePath + picklePrefix + \"_\" + \n",
        "        positions.lower() + \"_ACC.pickle\\n\\n\")\n",
        "  \n",
        "def extract_lacc_pickle(positions, picklePrefix='', unpicklePath='', \n",
        "                       picklePath='', prefix='lacc'):\n",
        "  # Extract LACC\n",
        "  print(\"Extracting LACC... \", end=\" \")\n",
        "  lacc = process_channels(positions, ['LAcc_x','LAcc_y','LAcc_z'], \n",
        "                          picklePrefix=picklePrefix, processEachAxis=True, \n",
        "                          statFeatures=True, spectralFeatures=True, \n",
        "                          process='rotate', unpicklePath=unpicklePath,\n",
        "                          prefix=prefix)\n",
        "  print(\"lacc Shape: \", lacc.shape)\n",
        "\n",
        "  # Pickle \n",
        "  import pickle as pk\n",
        "  with open(picklePath + picklePrefix + \"_\" + positions.lower() + \"_LACC.pickle\",\n",
        "            'wb') as f:\n",
        "    pk.dump(lacc,f)\n",
        "  print(\"\\nPickled at: \" + picklePath + picklePrefix + \"_\" + \n",
        "        positions.lower() + \"_LACC.pickle\\n\\n\")\n",
        "\n",
        "\n",
        "def extract_acch_pickle(positions, picklePrefix='', unpicklePath='', \n",
        "                       picklePath='', prefix='acch'):\n",
        "  # Extract ACCH\n",
        "  print(\"Extracting ACCH... \", end=\" \")\n",
        "  acch = process_channels(positions, ['LAcc_x','LAcc_y','LAcc_z'], \n",
        "                          picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                          statFeatures=True, spectralFeatures=True, \n",
        "                          process='horizontal', unpicklePath=unpicklePath,\n",
        "                          prefix=prefix)\n",
        "  print(\"acch Shape: \", acch.shape)\n",
        "\n",
        "  # Pickle \n",
        "  import pickle as pk\n",
        "  with open(picklePath + picklePrefix + \"_\" + positions.lower() + \"_ACCH.pickle\",\n",
        "            'wb') as f:\n",
        "    pk.dump(acch,f)\n",
        "  print(\"\\nPickled at: \" + picklePath + picklePrefix + \"_\" + \n",
        "        positions.lower() + \"_ACCH.pickle\\n\\n\")\n",
        "\n",
        "def extract_accv_pickle(positions, picklePrefix='', unpicklePath='', \n",
        "                       picklePath='', prefix='accv'):\n",
        "  # Extract ACCV\n",
        "  print(\"Extracting ACCV... \", end=\" \")\n",
        "  accv = process_channels(positions, ['LAcc_x','LAcc_y','LAcc_z'], \n",
        "                            picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                            statFeatures=True, spectralFeatures=True, \n",
        "                            process='vertical', unpicklePath=unpicklePath,\n",
        "                          prefix=prefix)\n",
        "  print(\"accv Shape: \", accv.shape)\n",
        "\n",
        "  # Pickle \n",
        "  import pickle as pk\n",
        "  with open(picklePath + picklePrefix + \"_\" + positions.lower() + \"_ACCV.pickle\",\n",
        "            'wb') as f:\n",
        "    pk.dump(accv,f)\n",
        "  print(\"\\nPickled at: \" + picklePath + picklePrefix + \"_\" + \n",
        "        positions.lower() + \"_ACCV.pickle\\n\\n\")\n",
        "\n",
        "\n",
        "def extract_jerkh_pickle(positions, picklePrefix='', unpicklePath='', \n",
        "                       picklePath='', prefix='jerkh'):\n",
        "  # Extract JERKH\n",
        "  print(\"Extracting JERKH... \", end=\" \")\n",
        "  jerkh = process_channels(positions, ['LAcc_x','LAcc_y','LAcc_z'], \n",
        "                          picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                          statFeatures=True, spectralFeatures=True, \n",
        "                          process='horizontal', unpicklePath=unpicklePath,\n",
        "                          calculateJerk=True, prefix=prefix)\n",
        "  print(\"jerkh Shape: \", jerkh.shape)\n",
        "\n",
        "  # Pickle \n",
        "  import pickle as pk\n",
        "  with open(picklePath + picklePrefix + \"_\" + positions.lower() + \"_JERKH.pickle\",\n",
        "            'wb') as f:\n",
        "    pk.dump(jerkh,f)\n",
        "  print(\"\\nPickled at: \" + picklePath + picklePrefix + \"_\" + \n",
        "        positions.lower() + \"_JERKH.pickle\\n\\n\")\n",
        "\n",
        "\n",
        "def extract_jerkv_pickle(positions, picklePrefix='', unpicklePath='', \n",
        "                       picklePath='', prefix='jerkv'):\n",
        "  # Extract JERKV\n",
        "  print(\"Extracting JERKV... \", end=\" \")\n",
        "  jerkv = process_channels(positions, ['LAcc_x','LAcc_y','LAcc_z'], \n",
        "                          picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                          statFeatures=True, spectralFeatures=True, \n",
        "                          process='vertical', unpicklePath=unpicklePath,\n",
        "                          calculateJerk=True, prefix=prefix)\n",
        "  print(\"jerkv Shape: \", jerkv.shape)\n",
        "\n",
        "  # Pickle \n",
        "  import pickle as pk\n",
        "  with open(picklePath + picklePrefix + \"_\" + positions.lower() + \"_JERKV.pickle\",\n",
        "            'wb') as f:\n",
        "    pk.dump(jerkv,f)\n",
        "  print(\"\\nPickled at: \" + picklePath + picklePrefix + \"_\" + \n",
        "        positions.lower() + \"_JERKV.pickle\\n\\n\")\n",
        "  \n",
        "\n",
        "def extract_mag_pickle(positions, picklePrefix='', unpicklePath='', \n",
        "                       picklePath='', prefix='mag'):\n",
        "  # Extract MAG\n",
        "  print(\"Extracting MAG... \", end=\" \")\n",
        "  mag = process_channels(positions, ['Mag_x','Mag_y','Mag_z'], \n",
        "                         picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                         statFeatures=True, spectralFeatures=True, \n",
        "                         process='rotate', unpicklePath=unpicklePath, \n",
        "                         prefix=prefix)\n",
        "  print(\"mag Shape: \", mag.shape)\n",
        "\n",
        "  # Pickle \n",
        "  import pickle as pk\n",
        "  with open(picklePath + picklePrefix + \"_\" + positions.lower() + \"_MAG.pickle\",\n",
        "            'wb') as f:\n",
        "    pk.dump(mag,f)\n",
        "  print(\"\\nPickled at: \" + picklePath + picklePrefix + \"_\" + \n",
        "        positions.lower() + \"_MAG.pickle\\n\\n\")\n",
        "\n",
        "\n",
        "def extract_gyr_pickle(positions, picklePrefix='', unpicklePath='', \n",
        "                       picklePath='', prefix='gyr'):\n",
        "  # Extract GYR\n",
        "  print(\"Extracting GYR... \", end=\" \")\n",
        "  gyr = process_channels(positions, ['Gyr_x','Gyr_y','Gyr_z'], \n",
        "                         picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                         statFeatures=True, spectralFeatures=True, \n",
        "                         unpicklePath=unpicklePath, prefix=prefix)\n",
        "  print(\"gyr Shape: \", gyr.shape)\n",
        "\n",
        "  # Pickle \n",
        "  import pickle as pk\n",
        "  with open(picklePath + picklePrefix + \"_\" + positions.lower() + \"_GYR.pickle\",\n",
        "            'wb') as f:\n",
        "    pk.dump(gyr,f)\n",
        "  print(\"\\nPickled at: \" + picklePath + picklePrefix + \"_\" + \n",
        "        positions.lower() + \"_GYR.pickle\\n\\n\")\n",
        "\n",
        "def extract_pres_pickle(positions, picklePrefix='', unpicklePath='', \n",
        "                       picklePath='', prefix='pres'):\n",
        "  # Extract PRES\n",
        "  print(\"Extracting PRES... \", end=\" \")\n",
        "  pres = process_channels(positions, ['Pressure'], picklePrefix=picklePrefix, \n",
        "                          statFeatures=True, spectralFeatures=True, \n",
        "                          unpicklePath=unpicklePath, prefix=prefix)\n",
        "  print(\"pres Shape: \", pres.shape)\n",
        "\n",
        "  # Pickle \n",
        "  import pickle as pk\n",
        "  with open(picklePath + picklePrefix + \"_\" + positions.lower() + \"_PRES.pickle\",\n",
        "            'wb') as f:\n",
        "    pk.dump(pres,f)\n",
        "  print(\"\\nPickled at: \" + picklePath + picklePrefix + \"_\" + \n",
        "        positions.lower() + \"_PRES.pickle\\n\\n\")\n",
        "\n",
        "\n",
        "def extract_label_pickle(positions, picklePrefix='', unpicklePath='', \n",
        "                       picklePath='', prefix='label'):\n",
        "  # Extract LABEL\n",
        "  print(\"Extracting LABEL... \", end=\" \")\n",
        "  label = process_channels(positions, ['Label'], picklePrefix=picklePrefix, \n",
        "                           isLabel=True, unpicklePath=unpicklePath, \n",
        "                           prefix=prefix)\n",
        "  print(\"Label Shape: \", label.shape)\n",
        "\n",
        "  # Pickle \n",
        "  import pickle as pk\n",
        "  with open(picklePath + picklePrefix + \"_\" + positions.lower() + \"_LABEL.pickle\",\n",
        "            'wb') as f:\n",
        "    pk.dump(label,f)\n",
        "  print(\"\\nPickled at: \" + picklePath + picklePrefix + \"_\" + \n",
        "        positions.lower() + \"_LABEL.pickle\\n\\n\")\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "def extractFeatures_pickle(positions, picklePrefix='', unpicklePath='', \n",
        "                           picklePath=''):\n",
        "  print(\"Importing data from: \" + picklePath + \"...\")\n",
        "  for j in positions:\n",
        "    print(\"\\nPosition: \" + j)\n",
        "  \n",
        "    # Extract Features\n",
        "    print(\"Calculating Features...\", end=\" \")\n",
        "    \n",
        "    acc = process_channels(j, ['Acc_x','Acc_y','Acc_z'], \n",
        "                           picklePrefix=picklePrefix, processEachAxis=True, \n",
        "                           statFeatures=True, spectralFeatures=True,  \n",
        "                           process='rotate', unpicklePath=unpicklePath, \n",
        "                           prefix='acc')\n",
        "    print(\"acc Shape: \", acc.shape)\n",
        "\n",
        "    lacc = process_channels(j, ['LAcc_x','LAcc_y','LAcc_z'], \n",
        "                            picklePrefix=picklePrefix, processEachAxis=True, \n",
        "                            statFeatures=True, spectralFeatures=True, \n",
        "                            process='rotate', unpicklePath=unpicklePath,\n",
        "                            prefix='lacc')\n",
        "    print(\"lacc Shape: \", lacc.shape)\n",
        "\n",
        "    acch = process_channels(j, ['LAcc_x','LAcc_y','LAcc_z'], \n",
        "                            picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                            statFeatures=True, spectralFeatures=True, \n",
        "                            process='horizontal', unpicklePath=unpicklePath, \n",
        "                            prefix='acch')\n",
        "    print(\"acch Shape: \", acch.shape)\n",
        "\n",
        "    accv = process_channels(j, ['LAcc_x','LAcc_y','LAcc_z'], \n",
        "                            picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                            statFeatures=True, spectralFeatures=True, \n",
        "                            process='vertical', unpicklePath=unpicklePath, \n",
        "                            prefix='accv')\n",
        "    print(\"accv Shape: \", accv.shape)\n",
        "\n",
        "    jerkh = process_channels(j, ['LAcc_x','LAcc_y','LAcc_z'], \n",
        "                            picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                            statFeatures=True, spectralFeatures=True, \n",
        "                            process='horizontal', unpicklePath=unpicklePath,\n",
        "                            calculateJerk=True, prefix='jerkh')\n",
        "    print(\"jerkh Shape: \", jerkh.shape)\n",
        "\n",
        "    jerkv = process_channels(j, ['LAcc_x','LAcc_y','LAcc_z'], \n",
        "                            picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                            statFeatures=True, spectralFeatures=True, \n",
        "                            process='vertical', unpicklePath=unpicklePath,\n",
        "                            calculateJerk=True, prefix='jerkv')\n",
        "    print(\"jerkv Shape: \", jerkv.shape)\n",
        "\n",
        "    mag = process_channels(j, ['Mag_x','Mag_y','Mag_z'], \n",
        "                           picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                           statFeatures=True, spectralFeatures=True, \n",
        "                           unpicklePath=unpicklePath, prefix='mag')\n",
        "    print(\"mag Shape: \", mag.shape)\n",
        "\n",
        "    gyr = process_channels(j, ['Gyr_x','Gyr_y','Gyr_z'], \n",
        "                           picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                           statFeatures=True, spectralFeatures=True, \n",
        "                           unpicklePath=unpicklePath, prefix='gyr')\n",
        "    print(\"gyr Shape: \", gyr.shape)\n",
        "\n",
        "    pres = process_channels(j, ['Pressure'], picklePrefix=picklePrefix, \n",
        "                            statFeatures=True, spectralFeatures=True, \n",
        "                            unpicklePath=unpicklePath, prefix='pres')\n",
        "    print(\"pres Shape: \", pres.shape)\n",
        "\n",
        "    print(\"\\nFeature Calculation Done\\n\")\n",
        "\n",
        "    print(\"Accumulating Features...\",end=\" \")\n",
        "    X = pd.concat([acc, lacc, acch, accv, jerkh, jerkv, mag, gyr, pres], axis=1)\n",
        "    print(\"Done | Shape: \", X.shape)\n",
        "\n",
        "    # Check for nan values in Features\n",
        "    #nan_count = np.isnan(X).sum()\n",
        "    #if nan_count != 0:\n",
        "    #  print(\"\\n\\tnan detected in X | \", nan_count, \"\\n\")\n",
        "\n",
        "    print(\"Processing Labels...\", end=\" \")\n",
        "    label = process_channels(j, ['Label'], picklePrefix=picklePrefix, \n",
        "                             isLabel=True, unpicklePath=unpicklePath, \n",
        "                             prefix='label')\n",
        "    print(\"Done | Shape: \", label.shape)\n",
        "    \n",
        "    # Check for nan values in Labels\n",
        "    #nan_count = np.isnan(label).sum()\n",
        "    #if nan_count != 0:\n",
        "    #  print(\"\\n\\tnan detected in y | \", nan_count, \"\\n\")\n",
        "\n",
        "    # Pickle Features\n",
        "    import pickle as pk\n",
        "    with open(picklePath + picklePrefix + \"_\" + j.lower() + \"_X.pickle\",\n",
        "              'wb') as f:\n",
        "      pk.dump(X,f)\n",
        "    print(\"\\n\\nPickled at: \" + picklePath + picklePrefix + \"_\" + j.lower() + \n",
        "          \"_X.pickle\")\n",
        "    \n",
        "    # Pickle Labels\n",
        "    import pickle as pk\n",
        "    with open(picklePath + picklePrefix + \"_\" + j.lower() + \"_y.pickle\",\n",
        "              'wb') as f:\n",
        "      pk.dump(label,f)\n",
        "    print(\"Pickled at: \" + picklePath + picklePrefix + \"_\" + j.lower() + \n",
        "          \"_y.pickle\")\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dubxo3SfZgbg"
      },
      "source": [
        "# DATASET PATH\n",
        "location_train = '/content/drive/My Drive/SHL Dataset 2020/Train/'\n",
        "location_cv = '/content/drive/My Drive/challenge-2020-validation/validation/'\n",
        "location_test = '/content/drive/My Drive/SHL 2020/TEST_2020/'\n",
        "#location_cv = '/content/drive/My Drive/validation_2020/'\n",
        "\n",
        "#positions = ['Bag','Hand','Torso','Hips']\n",
        "positions = ['Hand']\n",
        "\n",
        "channels = ['LAcc_x', 'LAcc_y', 'LAcc_z', 'Acc_x', 'Acc_y', 'Acc_z',\n",
        "            'Mag_x', 'Mag_y', 'Mag_z', 'Gyr_x', 'Gyr_y', 'Gyr_z',\n",
        "            'Gra_x', 'Gra_y', 'Gra_z', 'Ori_w', 'Ori_x', 'Ori_y', 'Ori_z',\n",
        "            'Pressure']#, 'Label']\n",
        "\n",
        "picklePrefix = 'test_2020'\n",
        "#picklePrefix = \"train_2020\"\n",
        "savePath = \"/content/drive/My Drive/test_2020/\"\n",
        "#savePath = \"/content/drive/My Drive/train_2020/\"\n",
        "#savePath = '/content/drive/My Drive/validation_2020/'\n",
        "temporaryPath = \"/content/\"\n",
        "#temporaryPath = \"/content/drive/My Drive/validation_2020/\"\n",
        "#picklePrefix = \"validation_2020\"\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZNERve3to19z"
      },
      "source": [
        "for pos in positions:\n",
        "\n",
        "  import_and_pickle(location_test, [pos], channels, picklePrefix=picklePrefix, \n",
        "                    picklePath=temporaryPath)\n",
        "\n",
        "  extract_acc_pickle(pos, picklePrefix=picklePrefix, unpicklePath=temporaryPath, \n",
        "                        picklePath=savePath)\n",
        "  extract_lacc_pickle(pos, picklePrefix=picklePrefix, unpicklePath=temporaryPath, \n",
        "                        picklePath=savePath)\n",
        "  extract_acch_pickle(pos, picklePrefix=picklePrefix, unpicklePath=temporaryPath, \n",
        "                        picklePath=savePath)\n",
        "  extract_accv_pickle(pos, picklePrefix=picklePrefix, unpicklePath=temporaryPath, \n",
        "                        picklePath=savePath)\n",
        "  extract_jerkh_pickle(pos, picklePrefix=picklePrefix, unpicklePath=temporaryPath, \n",
        "                        picklePath=savePath)\n",
        "  extract_jerkv_pickle(pos, picklePrefix=picklePrefix, unpicklePath=temporaryPath, \n",
        "                        picklePath=savePath)\n",
        "  extract_mag_pickle(pos, picklePrefix=picklePrefix, unpicklePath=temporaryPath, \n",
        "                        picklePath=savePath)\n",
        "  extract_gyr_pickle(pos, picklePrefix=picklePrefix, unpicklePath=temporaryPath, \n",
        "                        picklePath=savePath)\n",
        "  extract_pres_pickle(pos, picklePrefix=picklePrefix, unpicklePath=temporaryPath, \n",
        "                        picklePath=savePath)\n",
        "  extract_label_pickle(pos, picklePrefix=picklePrefix, unpicklePath=temporaryPath, \n",
        "                        picklePath=savePath)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QSWKtcvc4u0C"
      },
      "source": [
        "def accumulate_dataset(positions, picklePath='', picklePrefix='', addLabel=True, \n",
        "                       writeCSV=False):\n",
        "  \n",
        "  import pandas as pd\n",
        "\n",
        "  for pos in positions:\n",
        "  \n",
        "    print('Position: ', pos)\n",
        "\n",
        "    acc = pd.read_pickle(picklePath + picklePrefix + '_' + pos.lower() + '_' + \n",
        "                         'ACC.pickle')\n",
        "    lacc = pd.read_pickle(picklePath + picklePrefix + '_' + pos.lower() + '_' + \n",
        "                          'LACC.pickle')\n",
        "    acch = pd.read_pickle(picklePath + picklePrefix + '_' + pos.lower() + '_' + \n",
        "                          'ACCH.pickle')\n",
        "    accv = pd.read_pickle(picklePath + picklePrefix + '_' + pos.lower() + '_' + \n",
        "                          'ACCV.pickle')\n",
        "    jerkh = pd.read_pickle(picklePath + picklePrefix + '_' + pos.lower() + '_' + \n",
        "                           'JERKH.pickle')\n",
        "    jerkv = pd.read_pickle(picklePath + picklePrefix + '_' + pos.lower() + '_' + \n",
        "                           'JERKV.pickle')\n",
        "    mag = pd.read_pickle(picklePath + picklePrefix + '_' + pos.lower() + '_' + \n",
        "                         'MAG.pickle')\n",
        "    gyr = pd.read_pickle(picklePath + picklePrefix + '_' + pos.lower() + '_' + \n",
        "                         'GYR.pickle')\n",
        "    pres = pd.read_pickle(picklePath + picklePrefix + '_' + pos.lower() + '_' + \n",
        "                          'PRES.pickle')\n",
        "  \n",
        "    if addLabel==True:\n",
        "      label = pd.read_pickle(picklePath + picklePrefix + '_' + pos.lower() + \n",
        "                             '_' + 'LABEL.pickle')\n",
        "      data = pd.concat([acc, lacc, acch, accv, jerkh, jerkv, mag, gyr, pres, \n",
        "                        label], axis=1)\n",
        "    else:\n",
        "      data = pd.concat([acc, lacc, acch, accv, jerkh, jerkv, mag, gyr, pres], \n",
        "                       axis=1)\n",
        "\n",
        "    print('Accumulated Dataset | Shape: ', data.shape)\n",
        "\n",
        "    print('\\nPickling at: ' + picklePath + picklePrefix + '_' + pos.lower() + \n",
        "          '_' + 'DATA.pickle')\n",
        "    data.to_pickle(picklePath + picklePrefix + '_' + pos.lower() + '_' + \n",
        "                   'DATA.pickle')\n",
        "    if writeCSV==True:\n",
        "      data.to_csv(picklePath + picklePrefix + '_' + pos.lower() + '_' + \n",
        "                  'DATA.csv', index=False)\n",
        "    print('\\nDone\\n')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rbal2gFnGRMA",
        "outputId": "18961901-71ec-4809-ad54-f20259de7106",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        }
      },
      "source": [
        "#positions = ['Bag','Hand','Torso','Hips']\n",
        "positions = ['Hand']\n",
        "\n",
        "picklePrefix = 'test_2019'\n",
        "#picklePrefix = \"train_2020\"\n",
        "savePath = \"/content/drive/My Drive/test_2019/\"\n",
        "#savePath = \"/content/drive/My Drive/train_2020/\"\n",
        "#savePath = '/content/drive/My Drive/validation_2020/'\n",
        "#picklePrefix = \"validation_2020\"\n",
        "\n",
        "\n",
        "accumulate_dataset(positions, picklePath=savePath, picklePrefix=picklePrefix, \n",
        "                   addLabel=True, writeCSV=True)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Position:  Hand\n",
            "Accumulated Dataset | Shape:  (55811, 496)\n",
            "\n",
            "Pickling at: /content/drive/My Drive/test_2019/test_2019_hand_DATA.pickle\n",
            "\n",
            "Done\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2h3E3tUlul88"
      },
      "source": [
        "import_and_pickle(location_cv, ['Bag'], channels, picklePrefix=picklePrefix, \n",
        "                  picklePath=temporaryPath)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_nUJbmZLqwPP"
      },
      "source": [
        "extract_label_pickle('Bag', picklePrefix=picklePrefix, unpicklePath=temporaryPath, \n",
        "                       picklePath=savePath)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jeIMEvYb22kb"
      },
      "source": [
        "import pandas as pd\n",
        "nak = pd.read_pickle('/content/drive/My Drive/train_2020/train_2020_bag_MAG.pickle')\n",
        "print(type(nak))\n",
        "print(nak.shape)\n",
        "print(nak.head())\n",
        "print(list(nak.columns))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ApKhGKfnRbwv"
      },
      "source": [
        "extractFeatures_pickle(['Bag'], picklePrefix=picklePrefix, \n",
        "                        unpicklePath=temporaryPath, picklePath=savePath)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vPpp8C9Y4uwU"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b6fQhLxU4uql"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HCjfqLkl4uk3"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FVOnk-cv4udA"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ux6owfiO4uUL"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xu1vWfcK4uJH"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qQ4Aa7AGKmY9",
        "outputId": "9dbac7e4-909c-4be6-a0cb-23a5dc3afdd8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 248
        }
      },
      "source": [
        "picklePath = ''\n",
        "\n",
        "# Extract Features\n",
        "print(\"Calculating Features...\", end=\" \")\n",
        "\n",
        "acc = process_channels(positions[0], ['Acc_x','Acc_y','Acc_z'], \n",
        "                        picklePrefix=picklePrefix, processEachAxis=True, \n",
        "                        statFeatures=True, spectralFeatures=True, \n",
        "                        process='rotate', picklePath=picklePath)\n",
        "\n",
        "acch = process_channels(positions[0], ['LAcc_x','LAcc_y','LAcc_z'], \n",
        "                        picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                        statFeatures=True, spectralFeatures=True, \n",
        "                        process='horizontal', picklePath=picklePath)\n",
        "\n",
        "accv = process_channels(positions[0], ['LAcc_x','LAcc_y','LAcc_z'], \n",
        "                        picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                        statFeatures=True, spectralFeatures=True, \n",
        "                        process='vertical', picklePath=picklePath)\n",
        "\n",
        "mag = process_channels(positions[0], ['Mag_x','Mag_y','Mag_z'], \n",
        "                        picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                        statFeatures=True, spectralFeatures=True, \n",
        "                        picklePath=picklePath)\n",
        "\n",
        "gyr = process_channels(positions[0], ['Gyr_x','Gyr_y','Gyr_z'], \n",
        "                        picklePrefix=picklePrefix, processEachAxis=True,\n",
        "                        statFeatures=True, spectralFeatures=True, \n",
        "                        picklePath=picklePath)\n",
        "\n",
        "pres = process_channels(positions[0], ['Pressure'], picklePrefix=picklePrefix, \n",
        "                        statFeatures=True, spectralFeatures=True, \n",
        "                        picklePath=picklePath)\n",
        "print(\"Done\")\n",
        "\n",
        "label = process_channels(positions[0], ['Label'], picklePrefix=picklePrefix, \n",
        "                             isLabel=True, picklePath=picklePath)\n",
        "print(\"Done\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Calculating Features... "
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-2-1fb1f186b0fa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Calculating Features...\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\" \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m acc = process_channels(positions[0], ['Acc_x','Acc_y','Acc_z'], \n\u001b[0m\u001b[1;32m      7\u001b[0m                         \u001b[0mpicklePrefix\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpicklePrefix\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprocessEachAxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m                         \u001b[0mstatFeatures\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mspectralFeatures\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'process_channels' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iVu2IUbWfDFl",
        "outputId": "b3f5af60-ff1c-45f6-f0ee-6ab0de1fa4f0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "bag_X = np.concatenate((acc, acch, accv, mag, gyr, pres), \n",
        "                       axis=1)\n",
        "print(bag_X.shape)\n",
        "\n",
        "bag_y = label"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(28789, 231)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7F5xy9ZgNGlB"
      },
      "source": [
        "# Remove NaN rows\n",
        "\n",
        "def remove_nan_rows(X,indexfrom):\n",
        "  X = X[~np.isnan(indexfrom).any(axis=1)]\n",
        "  return X\n",
        "\n",
        "def cleanXy(X,y):\n",
        "  nan_count_x = np.isnan(X).sum()\n",
        "  print(nan_count_x,\" nan found in: X\")\n",
        "  nan_count_y = np.isnan(X).sum()\n",
        "  print(nan_count_y,\" nan found in: y\")\n",
        "\n",
        "  y = remove_nan_rows(y,X)\n",
        "  X = remove_nan_rows(X,X)\n",
        "\n",
        "  return X,y "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "emYydedgNoNE",
        "outputId": "7f90e1ac-f87c-4943-e83d-38e6c9e0d0eb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "X_train,y_train = cleanXy(bag_X,bag_y)\n",
        "#X_testC,y_testC = cleanXy(X_test,y_test)\n",
        "\n",
        "print(\"X_train Shape: \",X_train.shape)\n",
        "print(\"y_train Shape: \",y_train.shape)\n",
        "#print(\"X_test Shape: \",X_testC.shape)\n",
        "#print(\"y_test Shape: \",y_testC.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "4  nan found in: X\n",
            "4  nan found in: y\n",
            "X_train Shape:  (28788, 231)\n",
            "y_train Shape:  (28788,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h2BMg0-kOGzl",
        "outputId": "16456025-d910-4bba-8adb-952c5d0e27da",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 105
        }
      },
      "source": [
        "from sklearn.model_selection import train_test_split, cross_val_score\n",
        "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "\n",
        "\n",
        "#clf = GaussianNB()\n",
        "#clf = RandomForestClassifier(n_estimators=1000)\n",
        "#clf = LogisticRegression()\n",
        "#clf = DecisionTreeClassifier()\n",
        "#clf = KNeighborsClassifier(n_neighbors=3)\n",
        "#clf = SVC(kernel='linear', C=1)\n",
        "#clf = SVC(kernel='rbf', C=1, gamma=0.1)\n",
        "#clf = SVC(kernel='poly', degree=1, C=1)\n",
        "clf = AdaBoostClassifier()\n",
        "\n",
        "#train_test_split\n",
        "X_tr, X_test, y_tr, y_test = train_test_split(X_train, y_train, test_size = 0.20, random_state=1234)\n",
        "\n",
        "clf.fit(X_tr,y_tr)\n",
        "y_pred = clf.predict(X_test)\n",
        "print(\"Accuracy using train_test_split: %0.2f\" %(accuracy_score(y_test,y_pred)))\n",
        "print('\\n') \n",
        "\n",
        "'''\n",
        "print(\"cv = 3\")\n",
        "scores = cross_val_score(clf, X_train, y_train, cv=3,scoring='accuracy')\n",
        "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
        "print(\"Scores: \",scores)\n",
        "\n",
        "print(\"cv = 4\")\n",
        "scores = cross_val_score(clf, X_train, y_train, cv=4,scoring='accuracy')\n",
        "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
        "print(\"Scores: \",scores)\n",
        "\n",
        "print(\"cv = 6\")\n",
        "scores = cross_val_score(clf, X_train, y_train, cv=6,scoring='accuracy')\n",
        "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
        "print(\"Scores: \",scores)\n",
        "\n",
        "print(\"cv = 8\")\n",
        "scores = cross_val_score(clf, X_train, y_train, cv=8,scoring='accuracy')\n",
        "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
        "print(\"Scores: \",scores)\n",
        "\n",
        "print(\"cv = 10\")\n",
        "scores = cross_val_score(clf, X_train, y_train, cv=10,scoring='accuracy')\n",
        "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
        "print(\"Scores: \",scores)\n",
        "\n",
        "print(\"cv = 15\")\n",
        "scores = cross_val_score(clf, X_train, y_train, cv=15,scoring='accuracy')\n",
        "print(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\n",
        "print(\"Scores: \",scores)\n",
        "'''"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Accuracy using train_test_split: 0.49\n",
            "\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\nprint(\"cv = 3\")\\nscores = cross_val_score(clf, X_train, y_train, cv=3,scoring=\\'accuracy\\')\\nprint(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\\nprint(\"Scores: \",scores)\\n\\nprint(\"cv = 4\")\\nscores = cross_val_score(clf, X_train, y_train, cv=4,scoring=\\'accuracy\\')\\nprint(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\\nprint(\"Scores: \",scores)\\n\\nprint(\"cv = 6\")\\nscores = cross_val_score(clf, X_train, y_train, cv=6,scoring=\\'accuracy\\')\\nprint(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\\nprint(\"Scores: \",scores)\\n\\nprint(\"cv = 8\")\\nscores = cross_val_score(clf, X_train, y_train, cv=8,scoring=\\'accuracy\\')\\nprint(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\\nprint(\"Scores: \",scores)\\n\\nprint(\"cv = 10\")\\nscores = cross_val_score(clf, X_train, y_train, cv=10,scoring=\\'accuracy\\')\\nprint(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\\nprint(\"Scores: \",scores)\\n\\nprint(\"cv = 15\")\\nscores = cross_val_score(clf, X_train, y_train, cv=15,scoring=\\'accuracy\\')\\nprint(\"Accuracy: %0.2f (+/- %0.2f)\" % (scores.mean(), scores.std() * 2))\\nprint(\"Scores: \",scores)\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BQpkwm72BUIm",
        "outputId": "fb4bb08f-b124-41b4-a925-6d6e7e06cdaf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 285
        }
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import time\n",
        "from sklearn.model_selection import train_test_split, cross_val_score\n",
        "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
        "from sklearn.decomposition import PCA\n",
        "from sklearn.preprocessing import StandardScaler, LabelEncoder\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn import tree\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.ensemble import GradientBoostingClassifier, AdaBoostClassifier\n",
        "from sklearn.gaussian_process.kernels import RBF\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
        "from sklearn.gaussian_process import GaussianProcessClassifier\n",
        "\n",
        "\n",
        "dict_classifiers = {\n",
        "    \"Logistic Regression\": LogisticRegression(max_iter=1000,solver='saga'),\n",
        "    \"Nearest Neighbors\": KNeighborsClassifier(),\n",
        "    #\"RBF SVM\": SVC(C=10000,gamma=0.1),\n",
        "    #\"Linear SVM\": SVC(kernel='linear'),\n",
        "    #\"Gradient Boosting Classifier\": GradientBoostingClassifier(n_estimators=1000),\n",
        "    \"Decision Tree\": tree.DecisionTreeClassifier(),\n",
        "    #\"Random Forest\": RandomForestClassifier(n_estimators=1000),\n",
        "    \"Neural Net\": MLPClassifier(alpha = 1),\n",
        "    \"Naive Bayes\": GaussianNB(),\n",
        "    #\"AdaBoost\": AdaBoostClassifier(),\n",
        "    \"QDA\": QuadraticDiscriminantAnalysis(),\n",
        "    \"Gaussian Process\": GaussianProcessClassifier()\n",
        "}\n",
        "\n",
        "\n",
        "def batch_classify(X_train, Y_train, X_test, Y_test, no_classifiers = 5, verbose = True):\n",
        "    \"\"\"\n",
        "    This method, takes as input the X, Y matrices of the Train and Test set.\n",
        "    And fits them on all of the Classifiers specified in the dict_classifier.\n",
        "    The trained models, and accuracies are saved in a dictionary. The reason to use a dictionary\n",
        "    is because it is very easy to save the whole dictionary with the pickle module.\n",
        "\n",
        "    Usually, the SVM, Random Forest and Gradient Boosting Classifier take quiet some time to train.\n",
        "    So it is best to train them on a smaller dataset first and\n",
        "    decide whether you want to comment them out or not based on the test accuracy score.\n",
        "    \"\"\"\n",
        "\n",
        "    dict_models = {}\n",
        "    for classifier_name, classifier in list(dict_classifiers.items())[:no_classifiers]:\n",
        "        t_start = time.clock()\n",
        "        classifier.fit(X_train, Y_train)\n",
        "        t_end = time.clock()\n",
        "\n",
        "        t_diff = t_end - t_start\n",
        "        train_score = classifier.score(X_train, Y_train)\n",
        "        test_score = classifier.score(X_test, Y_test)\n",
        "\n",
        "        dict_models[classifier_name] = {'model': classifier, 'train_score': train_score, 'test_score': test_score, 'train_time': t_diff}\n",
        "        if verbose:\n",
        "            print(\"trained {c} in {f:.2f} s\".format(c=classifier_name, f=t_diff))\n",
        "    return dict_models\n",
        "\n",
        "\n",
        "\n",
        "def display_dict_models(dict_models, sort_by='test_score'):\n",
        "    cls = [key for key in dict_models.keys()]\n",
        "    test_s = [dict_models[key]['test_score'] for key in cls]\n",
        "    training_s = [dict_models[key]['train_score'] for key in cls]\n",
        "    training_t = [dict_models[key]['train_time'] for key in cls]\n",
        "\n",
        "    df_ = pd.DataFrame(data=np.zeros(shape=(len(cls),4)), columns = ['classifier', 'train_score', 'test_score', 'train_time'])\n",
        "    for ii in range(0,len(cls)):\n",
        "        df_.loc[ii, 'classifier'] = cls[ii]\n",
        "        df_.loc[ii, 'train_score'] = training_s[ii]\n",
        "        df_.loc[ii, 'test_score'] = test_s[ii]\n",
        "        df_.loc[ii, 'train_time'] = training_t[ii]\n",
        "\n",
        "    display(df_.sort_values(by=sort_by, ascending=False))\n",
        "\n",
        "\n",
        "# Train-Test Split\n",
        "X_tr, X_test, y_tr, y_test = train_test_split(X_train, y_train, test_size = 0.20, random_state=1234)\n",
        "\n",
        "dict_models = batch_classify(X_tr, y_tr, X_test, y_test, no_classifiers = 10)\n",
        "display_dict_models(dict_models)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-e8976553eaad>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m \u001b[0;31m# Train-Test Split\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m \u001b[0mX_tr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_tr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_test_split\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_size\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1234\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m \u001b[0mdict_models\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_classify\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_tr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_tr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mno_classifiers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'X_train' is not defined"
          ]
        }
      ]
    }
  ]
}