Preprocessing/PCA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 9 observations and 6 variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.random.randn(9,6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-1.30652685,  1.65813068, -0.11816405, -0.6801782 ,  0.66638308,\n",
       "        -0.46071979],\n",
       "       [-1.33425847, -1.34671751,  0.69377315, -0.15957344, -0.13370156,\n",
       "         1.07774381],\n",
       "       [-1.12682581, -0.73067775, -0.38487981,  0.09435159, -0.04217145,\n",
       "        -0.28688719],\n",
       "       [-0.0616264 , -0.10730528, -0.71960439, -0.81299299,  0.27451636,\n",
       "        -0.89091508],\n",
       "       [-1.15735526, -0.31229225, -0.15766702,  2.2567235 , -0.70470028,\n",
       "         0.94326072],\n",
       "       [ 0.74718833, -1.18894496,  0.77325298, -1.18388064, -2.65917224,\n",
       "         0.60631952],\n",
       "       [-1.75589058,  0.45093446, -0.6840109 ,  1.6595508 ,  1.0685094 ,\n",
       "        -0.4533858 ],\n",
       "       [-0.68783761, -1.2140774 , -0.44092263, -0.2803555 , -0.36469354,\n",
       "         0.15670386],\n",
       "       [ 0.5785215 ,  0.34965446, -0.76414392, -1.43779147,  1.36453185,\n",
       "        -0.68944918]])"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean = np.mean(data,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-6.78290128e-01, -2.71255061e-01, -2.00262954e-01, -6.04607063e-02,\n",
       "       -5.89442647e-02,  2.96762156e-04])"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "zero_mean = np.zeros((9,6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(9):\n",
    "    zero_mean[i,:] = data[i,:] -  mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.62823672,  1.92938574,  0.08209891, -0.6197175 ,  0.72532735,\n",
       "        -0.46101655],\n",
       "       [-0.65596834, -1.07546245,  0.89403611, -0.09911273, -0.07475729,\n",
       "         1.07744704],\n",
       "       [-0.44853568, -0.45942269, -0.18461686,  0.1548123 ,  0.01677281,\n",
       "        -0.28718395],\n",
       "       [ 0.61666373,  0.16394978, -0.51934143, -0.75253228,  0.33346062,\n",
       "        -0.89121185],\n",
       "       [-0.47906513, -0.04103719,  0.04259594,  2.3171842 , -0.64575601,\n",
       "         0.94296396],\n",
       "       [ 1.42547846, -0.91768989,  0.97351593, -1.12341993, -2.60022797,\n",
       "         0.60602276],\n",
       "       [-1.07760046,  0.72218952, -0.48374794,  1.7200115 ,  1.12745366,\n",
       "        -0.45368257],\n",
       "       [-0.00954748, -0.94282234, -0.24065968, -0.21989479, -0.30574928,\n",
       "         0.15640709],\n",
       "       [ 1.25681163,  0.62090952, -0.56388097, -1.37733077,  1.42347611,\n",
       "        -0.68974595]])"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zero_mean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## c = xx^T/(n-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.80110198, -0.18460298,  0.03834045, -0.79663478, -0.37859036,\n",
       "        -0.10065795],\n",
       "       [-0.18460298,  0.96961708, -0.27139272,  0.03082184,  0.7407319 ,\n",
       "        -0.445078  ],\n",
       "       [ 0.03834045, -0.27139272,  0.33365979, -0.0968368 , -0.50211581,\n",
       "         0.33027385],\n",
       "       [-0.79663478,  0.03082184, -0.0968368 ,  1.56492414,  0.09753111,\n",
       "         0.30557456],\n",
       "       [-0.37859036,  0.7407319 , -0.50211581,  0.09753111,  1.40153384,\n",
       "        -0.55535277],\n",
       "       [-0.10065795, -0.445078  ,  0.33027385,  0.30557456, -0.55535277,\n",
       "         0.52658084]])"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.dot(zero_mean.T, zero_mean)/(9 - 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## np.cov() takes (m,n), m represents # of variables\n",
    "## if set row_var = 0, then it means m represents # of observations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.80110198, -0.18460298,  0.03834045, -0.79663478, -0.37859036,\n",
       "        -0.10065795],\n",
       "       [-0.18460298,  0.96961708, -0.27139272,  0.03082184,  0.7407319 ,\n",
       "        -0.445078  ],\n",
       "       [ 0.03834045, -0.27139272,  0.33365979, -0.0968368 , -0.50211581,\n",
       "         0.33027385],\n",
       "       [-0.79663478,  0.03082184, -0.0968368 ,  1.56492414,  0.09753111,\n",
       "         0.30557456],\n",
       "       [-0.37859036,  0.7407319 , -0.50211581,  0.09753111,  1.40153384,\n",
       "        -0.55535277],\n",
       "       [-0.10065795, -0.445078  ,  0.33027385,  0.30557456, -0.55535277,\n",
       "         0.52658084]])"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.cov(data.T)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Same!!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "covariance = np.cov(data.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.80110198, -0.18460298,  0.03834045, -0.79663478, -0.37859036,\n",
       "        -0.10065795],\n",
       "       [-0.18460298,  0.96961708, -0.27139272,  0.03082184,  0.7407319 ,\n",
       "        -0.445078  ],\n",
       "       [ 0.03834045, -0.27139272,  0.33365979, -0.0968368 , -0.50211581,\n",
       "         0.33027385],\n",
       "       [-0.79663478,  0.03082184, -0.0968368 ,  1.56492414,  0.09753111,\n",
       "         0.30557456],\n",
       "       [-0.37859036,  0.7407319 , -0.50211581,  0.09753111,  1.40153384,\n",
       "        -0.55535277],\n",
       "       [-0.10065795, -0.445078  ,  0.33027385,  0.30557456, -0.55535277,\n",
       "         0.52658084]])"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "covariance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from numpy import linalg as LA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The normalized (unit “length”) eigenvectors, such that the column v[:,i] is the eigenvector corresponding to the eigenvalue w[i]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "eigenValue, eigenVector = LA.eig(covariance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2.56248593, 2.08105852, 0.01673354, 0.1396961 , 0.36172544,\n",
       "       0.43571813])"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eigenValue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.30592151,  0.4287534 , -0.19011475, -0.56366791,  0.60319613,\n",
       "         0.06979868],\n",
       "       [ 0.47844192,  0.19238731,  0.13491458, -0.13942573, -0.07801042,\n",
       "         0.83087505],\n",
       "       [-0.26858554, -0.09515344, -0.7574641 , -0.19015493, -0.51045315,\n",
       "         0.21985098],\n",
       "       [ 0.26625865, -0.78978374, -0.24268048, -0.15737782,  0.46291276,\n",
       "         0.08601257],\n",
       "       [ 0.6763357 ,  0.19619372, -0.16764073, -0.43384925, -0.19695834,\n",
       "        -0.49895577],\n",
       "       [-0.27756714, -0.32841002,  0.53376526, -0.64318513, -0.34121192,\n",
       "         0.00923643]])"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eigenVector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## covariance . v = lambda . v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.78391956,  0.89226092, -0.00318129, -0.07874221,  0.21819138,\n",
       "         0.03041255],\n",
       "       [ 1.22600068,  0.40036924,  0.0022576 , -0.01947723, -0.02821835,\n",
       "         0.36202732],\n",
       "       [-0.68824667, -0.19801987, -0.01267506, -0.0265639 , -0.18464389,\n",
       "         0.09579306],\n",
       "       [ 0.68228404, -1.64358617, -0.0040609 , -0.02198507,  0.16744732,\n",
       "         0.03747724],\n",
       "       [ 1.73310073,  0.40829062, -0.00280522, -0.06060705, -0.07124484,\n",
       "        -0.21740408],\n",
       "       [-0.7112619 , -0.68344046,  0.00893178, -0.08985046, -0.12342503,\n",
       "         0.00402448]])"
      ]
     },
     "execution_count": 173,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.dot(covariance, eigenVector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.78391956,  0.89226092, -0.00318129, -0.07874221,  0.21819138,\n",
       "         0.03041255],\n",
       "       [ 1.22600068,  0.40036924,  0.0022576 , -0.01947723, -0.02821835,\n",
       "         0.36202732],\n",
       "       [-0.68824667, -0.19801987, -0.01267506, -0.0265639 , -0.18464389,\n",
       "         0.09579306],\n",
       "       [ 0.68228404, -1.64358617, -0.0040609 , -0.02198507,  0.16744732,\n",
       "         0.03747724],\n",
       "       [ 1.73310073,  0.40829062, -0.00280522, -0.06060705, -0.07124484,\n",
       "        -0.21740408],\n",
       "       [-0.7112619 , -0.68344046,  0.00893178, -0.08985046, -0.12342503,\n",
       "         0.00402448]])"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eigenValue * eigenVector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9, 6)"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zero_mean.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6, 6)"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eigenVector.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Choose three principle components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 230,
   "metadata": {},
   "outputs": [],
   "source": [
    "Vect = eigenVector[:,:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.30592151,  0.4287534 ],\n",
       "       [ 0.47844192,  0.19238731],\n",
       "       [-0.26858554, -0.09515344],\n",
       "       [ 0.26625865, -0.78978374],\n",
       "       [ 0.6763357 ,  0.19619372],\n",
       "       [-0.27756714, -0.32841002]])"
      ]
     },
     "execution_count": 231,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Vect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## finaldata = zero_mean * choosen_eigen_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [],
   "source": [
    "FinalData = np.dot(zero_mean, Vect)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.54676224,  0.87716863],\n",
       "       [-0.9300112 , -0.86345828],\n",
       "       [ 0.09927205, -0.28779479],\n",
       "       [ 0.30181157,  1.29779919],\n",
       "       [ 0.03396844, -2.48379548],\n",
       "       [-3.36257743,  0.52007991],\n",
       "       [ 2.15154708, -1.26529836],\n",
       "       [-0.69227875, -0.10026339],\n",
       "       [ 0.85150601,  2.30556258]])"
      ]
     },
     "execution_count": 233,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FinalData"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Map the data to the vector's direction. Now this FinalData is in a new space and with lower dimension."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Go back to original data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.794453  ,  0.65106572, -0.77512252, -0.36573059,  1.14247079,\n",
       "        -0.66358037],\n",
       "       [-0.74765996, -0.89391866,  0.19675003,  0.39470676, -0.8429492 ,\n",
       "         0.49615647],\n",
       "       [-0.82624658, -0.28324744, -0.17640877,  0.20067832, -0.0431469 ,\n",
       "         0.05095588],\n",
       "       [-0.20454429,  0.11598285, -0.3664044 , -0.99277518,  0.40830276,\n",
       "        -0.53675358],\n",
       "       [-1.77310756, -0.71902282, -0.05069764,  1.88536609, -0.5404613 ,\n",
       "         0.86129155],\n",
       "       [ 0.59243158, -1.79351573,  0.72929291, -1.34220828, -2.21434017,\n",
       "         0.70935104],\n",
       "       [-1.84480216,  0.49044256, -0.52150466,  1.55536717,  1.17813161,\n",
       "        -0.27736679],\n",
       "       [-0.55607556, -0.58870414, -0.1903732 , -0.2250588 , -0.5879019 ,\n",
       "         0.35615623],\n",
       "       [ 0.04984637,  0.5796221 , -0.64789834, -1.65449184,  0.96939594,\n",
       "        -0.99353958]])"
      ]
     },
     "execution_count": 222,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.dot(FinalData,  Vect.T) + mean"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wrap up PCA as a function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def pca(dt, n):\n",
    "    mean = np.mean(dt,axis=0)\n",
    "    #calculate zero mean matrix\n",
    "    data_adjust = dt - mean\n",
    "    #calculate covariance matrix\n",
    "    covariance = np.cov(data_adjust, rowvar = 0)\n",
    "    #calculate eigen values and eigen vectors\n",
    "    eigenVals, eigenVects = np.linalg.eig(covariance)\n",
    "    #get the indices of n biggest eigen values\n",
    "    eigValIndice = np.argsort(eigenVals)[-n:][::-1]\n",
    "    select_vects = eigenVects[:,eigValIndice]\n",
    "    final_data = np.dot(data_adjust, select_vects)\n",
    "    recon_dt = np.dot(final_data, select_vects.T) + mean\n",
    "    return final_data, recon_dt"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}