Chapter_2_sec_3.1_3.5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# 2.3 Lab: Introduction to R"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "## 2.3.1 Basic Commands"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "# best practice is to have all the modules imported at the top of the file, but for this one, I will import them when we need them\n",
    "import numpy as np  # for calculation purpose, let us use np.array \n",
    "import random # for the random number generation\n",
    "\n",
    "x = np.array([1, 3, 2, 5])\n",
    "# use print to see the array\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [],
   "source": [
    "x = np.array([1, 6, 2])\n",
    "print(x)\n",
    "y = [1, 4, 3]\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use len() to find length of a vector\n",
    "len(x)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(x + y) # please note that we define x and y a little bit differently, but we still can do the calculation "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The whos function allows us to look at a list of all of the objects, such as data and functions, that we have saved so far\n",
    "%whos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reset_selective x\n",
    "del x "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%whos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read the description of a function \n",
    "%whos?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get a matrix \n",
    "x = [[1,2],[3, 4]]\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we could also reshape a one dimensional array to a matrix\n",
    "x = np.array([1, 2, 3, 4])\n",
    "print(x)\n",
    "x = np.reshape(x, [2,2])\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# then we can use the matrix to do some calculations\n",
    "np.sqrt(x)\n",
    "x**2\n",
    "np.square(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use random to generate random numbers/arrays/matrices\n",
    "mu, sigma = 0, 1\n",
    "x = np.random.normal(mu, sigma, 5)\n",
    "y = x + np.random.normal(20, 0.1, 5)\n",
    "print(x)\n",
    "print(y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# more calculation\n",
    "np.corrcoef(x, y) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# above will return the correlation matrix, let us see just the correlation coefficient between x and y\n",
    "np.corrcoef(x, y)[0,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# we can use the seed function to set up the random seed, so that every thing we run the code, we will get the same result\n",
    "random.seed(2333)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# after set up the seed, this should genernate the same result everytime we run the notebook\n",
    "np.random.normal(mu, sigma, 5) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# you could increase the number of samples to see the empirical distribution coverages to the theoretical distribution\n",
    "mu, sigma = 0, 1\n",
    "num_samples = 10\n",
    "x = np.random.normal(mu, sigma, num_samples)\n",
    "print(np.mean(x))\n",
    "print(np.var(x))\n",
    "print(np.sqrt(np.var(x)))\n",
    "print(np.std(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3.2 Graphics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np  # for calculation purpose, let use np.array \n",
    "import random # for the random \n",
    "\n",
    "x = np.random.normal(0, 1, 100)\n",
    "y = np.random.normal(0, 1, 100)\n",
    "\n",
    "# in python, matplotlib is the most used library for plot \n",
    "# matplotlib.pyplot is a collection of command style functions that make matplotlib work like MATLAB.\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "plt.plot(x, y, 'bo') # please use plt.plot? to look at more options \n",
    "plt.ylabel(\"this is the y-axis\")\n",
    "plt.xlabel(\"this is the x-axis\")\n",
    "plt.title(\"Plot of X vs Y\")\n",
    "plt.savefig('Figure.pdf') # use plt.savefig function to save images\n",
    "plt.show() \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# note the arange excludes right end of rande specification \n",
    "x = np.arange(1, 11) \n",
    "print(x) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# note: np.arange actually can result in unexpected results; check np.arange(0.2, 0.6, 0.4) vs np.arange(0.2, 1.6, 1.4)\n",
    "print(np.arange(0.2, 0.6, 0.4))\n",
    "print(np.arange(0.2, 1.6, 1.4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# in order to use Pi, math module needs to loaded first\n",
    "import math\n",
    "x = np.linspace(-math.pi, math.pi, num = 50)\n",
    "print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.cm as cm\n",
    "import matplotlib.mlab as mlab\n",
    "y = x\n",
    "X, Y = np.meshgrid(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%whos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# same as above, use plt.contour? to explore the options\n",
    "f = np.cos(Y)/(1 + np.square(X))\n",
    "CS = plt.contour(X, Y, f)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I think imshow looks nicer for heatmap, use 'extent =' fix the x, y axis\n",
    "fa = (f - f.T)/2 #f.T for transpose or tranpose(f)\n",
    "plt.imshow(fa, extent=(x[0], x[-1], y[0], y[-1])) \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from mpl_toolkits.mplot3d import axes3d\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111, projection='3d')\n",
    "ax.plot_wireframe(X, Y, fa)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3.3 Indexing Data \n",
    "Here we use np array. If the data structure is something else, the method below may not work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "A = np.arange(1,17,1).reshape(4, 4).transpose()\n",
    "print(A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# one thing to note here is that in python, the index starts from 0, not 1\n",
    "print(A[2, 3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# try the same index as the book, but we got different number. The reason is R starts the index from 1 (Matlab too), but Python starts the index from 0. To select the same number (10) as the book did, we reduce the index by 1\n",
    "print(A[1, 2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to select a submatrix, need the non-singleton dimension of your indexing array to be aligned with the axis you're indexing into, \n",
    "# e.g. for an n x m 2D subarray: A[n by 1 array,1 by m array]\n",
    "A[[[0],[2]], [1,3]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this is another way to do that\n",
    "A[0:3:2, 1:4:2] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select all columns in those two rows \n",
    "A[0:3:2,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select all row in those two columns \n",
    "A[:, 1:4:2] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the last two examples include either no index for the columns or no index for the rows. These indicate that Python should include all columns or all rows, respectively\n",
    "A[0,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# '-' sign has a different meaning and good usage in Python. This means index from the end, -1 means the last element \n",
    "A[-1, -1] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# there are other ways to let Python keep all rows except certain index. For example, we could also use boolean. \n",
    "ind = np.ones((4,), bool)\n",
    "ind[[0,2]] = False\n",
    "print(ind)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "A[ind,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we do not specify the row or column, the default is the for the row\n",
    "A[ind]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we use .shape to get the shape of the matrix \n",
    "A.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 2.3.4 Loading Data\n",
    "\n",
    "In Python, Pandas is a common used module to read from file into a data frame. I downloaded the Auto.csv from the book website. First, take a look at the csv file. There are headers, missing value is marked by '?'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "Auto = pd.read_csv('data/Auto.csv', header=0, na_values='?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we could use .head to see the first few rows (default = 5) of the data \n",
    "Auto.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check one record with missing value, and make sure the missing value is correctly imported. \n",
    "# Here we use the i.loc to select the row which is different from the indexing method above\n",
    "# the reason is that Auto is a pandas dataframe, while the indexing method was for a numpy array\n",
    "Auto.iloc[32]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the same .shape function as in ndarray to find out the dimension of the data frame \n",
    "Auto.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# an alternative way to select the first 4 rows. \n",
    "Auto[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# an alternative way to select the first 4 rows and first 2 columns.\n",
    "Auto.iloc[:4, :2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we can use list to find the column names or use .columns\n",
    "print(list(Auto))\n",
    "print(Auto.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use .isnull and .sum to find out how many NaNs in each variables\n",
    "Auto.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# after the previous steps, there are 397 obs in the data and only 5 with missing values. We can just drop the ones with missing values  \n",
    "print(Auto.shape)\n",
    "Auto = Auto.dropna()\n",
    "print(Auto.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3.5 Additional Graphical and Numerical Summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# refer a column of data frame by name, by using a '.'. Ref the options in plt.plot for more.\n",
    "plt.plot(Auto.cylinders, Auto.mpg, 'ro')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use .hist to get the histogram of certain variables. column = to specify which variable\n",
    "Auto.hist(column = ['cylinders', 'mpg'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use the .describe() to get a summary of the data frame. Use .describe ( include = 'all' ) for mix types, use describe(include = [np.number]) for numerical columns, use describe(include = ['O']) for objects.\n",
    "Auto.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# we can change type of certain variable(s). Here changed the cylinders into categorical variable \n",
    "Auto['cylinders'] = Auto['cylinders'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Auto.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Auto.describe(include= 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# End of Chapter 2"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "interpreter": {
   "hash": "4548a0e672c5b3a287feee7b2962606840aa548749d1830ef724408652b0c250"
  },
  "kernelspec": {
   "display_name": "Python 2.7.16 64-bit ('base': conda)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}