data_extract.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reading Data from csv file and doing basic data manipulation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.read_csv(\"1.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get data for individual Trips"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for tripID in set(df['tripID'].values):\n",
    "    df_trip = df[df['tripID']==tripID]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Converting Raw ECompass data to engineering units"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "# Convert Accelerometer reading to g units, 8bit data with a range of +-2g #\n",
    "def convert_acc(x):\n",
    "    x = int(x,16)\n",
    "    if x>127:\n",
    "        x = x-256\n",
    "    return np.float64(x*2/128)\n",
    "\n",
    "# Convert Magnetometer reading to uT units, 16bit data with a range of +-1200uT #\n",
    "def convert_mag(x):\n",
    "    x = int(x,16)\n",
    "    if x>32767:\n",
    "        x = x-65536\n",
    "    return np.float64(x*0.0366)\n",
    "\n",
    "\n",
    "# Convert a single row of Accelerometer data to engineering units and return a dict #\n",
    "def convert_acc_row(row):\n",
    "    # Initially the data was gathered without magnetometer, so check length for identification #\n",
    "    data_dict = {'x': [], 'y': [], 'z': []}\n",
    "    if len(row) == 150:\n",
    "        for i in range(0, len(row), 6):\n",
    "            data_dict['x'].append(convert_acc(row[i:i+2]))\n",
    "            data_dict['y'].append(convert_acc(row[i+2:i+4]))\n",
    "            data_dict['z'].append(convert_acc(row[i+4:i+6]))\n",
    "    return data_dict\n",
    "\n",
    "\n",
    "# Convert a single row of Accelerometer and Magnetometer data to engineering units and return a df #\n",
    "def convert_acc_mag_row(row):\n",
    "    # Initially the data was gathered without magnetometer, so check length for identification #\n",
    "    data_dict = {'x': [], 'y': [], 'z': []}\n",
    "    if len(row) == 162:\n",
    "        data_dict['mx'] = convert_mag(row[:4])\n",
    "        data_dict['my'] = convert_mag(row[4:8])\n",
    "        data_dict['mz'] = convert_mag(row[8:12])\n",
    "        row = row[12:]\n",
    "        for i in range(0, len(row), 6):\n",
    "            data_dict['x'].append(convert_acc(row[i:i+2]))\n",
    "            data_dict['y'].append(convert_acc(row[i+2:i+4]))\n",
    "            data_dict['z'].append(convert_acc(row[i+4:i+6]))\n",
    "    return data_list\n",
    "\n",
    "df_trip['accData'] = df_trip['accData'].apply(convert_acc_row)\n",
    "# Drop the values for first second as the data is not collected completely #\n",
    "df_trip = df_trip.reset_index(drop=True)\n",
    "df_trip = df_trip.drop(0).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculating Rotation Matrix for vehicle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We need data for the device when the vehicle is static #\n",
    "[mx,my,mz] = df_trip[['mx', 'my', 'mz']][df_trip['speed'] <= 1].mean().tolist()\n",
    "[x,y,z] = df_trip['accData'][df_trip['speed'] <= 1].iloc[0].mean().tolist()\n",
    "\n",
    "import math\n",
    "phi = math.atan(y/z)\n",
    "theta = math.atan(-x/(y*math.sin(phi)+z*math.cos(phi)))\n",
    "psi = math.atan((mz*math.sin(phi) - my*math.cos(phi))/(mx*math.cos(theta) + my*math.sin(theta)*math.sin(phi) + mz*math.sin(theta)*math.cos(phi)))\n",
    "\n",
    "Rx = np.array([[1,0,0], [0,math.cos(phi),math.sin(phi)], [0,-math.sin(phi),math.cos(phi)]])\n",
    "Ry = np.array([[math.cos(theta),0,-math.sin(theta)], [0,1,0], [math.sin(theta),0,math.cos(theta)]])\n",
    "Rz = np.array([[math.cos(phi),math.sin(phi),0], [-math.sin(phi),math.cos(phi),0], [0,0,1]])\n",
    "\n",
    "R = Rx.dot(Ry).dot(Rz)\n",
    "R = np.linalg.inv(R)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Rotate Accelerometer values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def rotate_acc(row):\n",
    "    A = np.array([[row['ax']], [row['ay']], [row['az']]])\n",
    "    B = np.dot(R,A)\n",
    "    return B[:,0]\n",
    "\n",
    "acc_sample = df_trip['accData'].iloc[0]\n",
    "acc_sample.apply(rotate_acc, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Storing DataFrame in hdf file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_hdf(\"data.hdf5\", \"first\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}