Step3_Machine+Learning+Implementation+and+Optimization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# First we import all our required libraries for data manipulation and ML implementation\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import svm\n",
    "from sklearn import tree\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# In this new workbook we need to import our datasets again and get our target\n",
    "reduced_megaset = pd.read_csv('reduced_megaset.csv')\n",
    "SP500 = pd.read_csv('SP500_2.csv', parse_dates=True)\n",
    "target_raw = (SP500['Adj Close'].shift(-1)/SP500['Adj Close'])-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our target has 6232 values\n"
     ]
    }
   ],
   "source": [
    "# Label encode our target variable, 1 for increase, 0 for decrease/no change\n",
    "target = target_raw[21:]\n",
    "target[target > 0] = 1\n",
    "target[target <= 0] = 0\n",
    "print(\"Our target has {} values\".format(target.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our training set has shape 6001, 10\n",
      "Our training set target is 6001 long\n",
      "Our testing set has shape 230, 10\n",
      "Our testing set target is 230 long\n"
     ]
    }
   ],
   "source": [
    "# We split our megaset into training and cross-validation (test) subsets\n",
    "X_train = reduced_megaset[:6001]\n",
    "X_test = reduced_megaset[6001:-1]\n",
    "y_train = target[:6001]\n",
    "y_test = target[6001:-1]\n",
    "print(\"Our training set has shape {}, {}\".format(X_train.shape[0], X_train.shape[1]))\n",
    "print(\"Our training set target is {} long\".format(y_train.shape[0]))\n",
    "print(\"Our testing set has shape {}, {}\".format(X_test.shape[0], X_test.shape[1]))\n",
    "print(\"Our testing set target is {} long\".format(y_test.shape[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SVM Linear: [Accuracy: 0.5826, f1-score: 0.7363]\n",
      "SVM RBF: [Accuracy: 0.5826, f1-score: 0.7363]\n"
     ]
    }
   ],
   "source": [
    "# We train SVM classifiers with two different kernels\n",
    "\"\"\"Support Vector Classifier with Linear Kernel\"\"\"\n",
    "clf1 = svm.SVC(kernel = 'linear')\n",
    "clf1.fit(X_train, y_train)\n",
    "clf1_predictions = clf1.predict(X_test)\n",
    "clf1_accuracy = accuracy_score(y_test, clf1_predictions)\n",
    "clf1_f1 = f1_score(y_test, clf1_predictions)\n",
    "print(\"SVM Linear: [Accuracy: {:.4f}, f1-score: {:.4f}]\".format(clf1_accuracy, clf1_f1))\n",
    "\n",
    "\"\"\"Support Vector Classifier with RBF Kernel\"\"\"\n",
    "clf2 = svm.SVC(kernel = 'rbf')\n",
    "clf2.fit(X_train, y_train)\n",
    "clf2_predictions = clf2.predict(X_test)\n",
    "clf2_accuracy = accuracy_score(y_test, clf2_predictions)\n",
    "clf2_f1 = f1_score(y_test, clf2_predictions)\n",
    "print(\"SVM RBF: [Accuracy: {:.4f}, f1-score: {:.4f}]\".format(clf2_accuracy, clf2_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kNN: [Accuracy: 0.5391, f1-score: 0.5760]\n"
     ]
    }
   ],
   "source": [
    "# We 'train' a k-Nearest Neighbours model\n",
    "clf3 = KNeighborsClassifier(n_neighbors = 3)\n",
    "clf3.fit(X_train, y_train)\n",
    "clf3_predictions = clf3.predict(X_test)\n",
    "clf3_accuracy = accuracy_score(y_test, clf3_predictions)\n",
    "clf3_f1 = f1_score(y_test, clf3_predictions)\n",
    "print(\"kNN: [Accuracy: {:.4f}, f1-score: {:.4f}]\".format(clf3_accuracy, clf3_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision Tree: [Accuracy: 0.4870, f1-score: 0.5317]\n",
      "Random Forest Classifier: [Accuracy: 0.4870, f1-score: 0.5317]\n"
     ]
    }
   ],
   "source": [
    "# We train our decision tree and random forest classifiers\n",
    "\"\"\"Decision Tree Classifier\"\"\"\n",
    "clf4 = tree.DecisionTreeClassifier()\n",
    "clf4.fit(X_train, y_train)\n",
    "clf4_predictions = clf4.predict(X_test)\n",
    "clf4_accuracy = accuracy_score(y_test, clf4_predictions)\n",
    "clf4_f1 = f1_score(y_test, clf4_predictions)\n",
    "print(\"Decision Tree: [Accuracy: {:.4f}, f1-score: {:.4f}]\".format(clf4_accuracy, clf4_f1))\n",
    "\n",
    "\"\"\"Random Forest Classifier\"\"\"\n",
    "clf5 = RandomForestClassifier(n_estimators=10)\n",
    "clf5.fit(X_train, y_train)\n",
    "clf5_predictions = clf4.predict(X_test)\n",
    "clf5_accuracy = accuracy_score(y_test, clf5_predictions)\n",
    "clf5_f1 = f1_score(y_test, clf5_predictions)\n",
    "print(\"Random Forest Classifier: [Accuracy: {:.4f}, f1-score: {:.4f}]\".format(clf5_accuracy, clf5_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoost Classifier DT: [Accuracy: 0.5913, f1-score: 0.7152]\n",
      "AdaBoost Classifier NB: [Accuracy: 0.5783, f1-score: 0.7298]\n"
     ]
    }
   ],
   "source": [
    "# We train our AdaBoost Classifier with two different base estimators\n",
    "\"\"\"AdaBoost Classifier with DecisionTree base\"\"\"\n",
    "clf6a = AdaBoostClassifier(n_estimators=100)\n",
    "clf6a.fit(X_train, y_train)\n",
    "clf6a_predictions = clf6a.predict(X_test)\n",
    "clf6a_accuracy = accuracy_score(y_test, clf6a_predictions)\n",
    "clf6a_f1 = f1_score(y_test, clf6a_predictions)\n",
    "print(\"AdaBoost Classifier DT: [Accuracy: {:.4f}, f1-score: {:.4f}]\".format(clf6a_accuracy, clf6a_f1))\n",
    "\n",
    "\"\"\"AdaBoost Classifier with NaiveBayes base\"\"\"\n",
    "clf6b = AdaBoostClassifier(n_estimators=100, base_estimator=GaussianNB())\n",
    "clf6b.fit(X_train, y_train)\n",
    "clf6b_predictions = clf6b.predict(X_test)\n",
    "clf6b_accuracy = accuracy_score(y_test, clf6b_predictions)\n",
    "clf6b_f1 = f1_score(y_test, clf6b_predictions)\n",
    "print(\"AdaBoost Classifier NB: [Accuracy: {:.4f}, f1-score: {:.4f}]\".format(clf6b_accuracy, clf6b_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting Classifier: [Accuracy: 0.5348, f1-score: 0.6748]\n"
     ]
    }
   ],
   "source": [
    "# We train our Gradient Boosting Classifier\n",
    "\"\"\"Gradient Boosting Classifier\"\"\"\n",
    "clf7 = GradientBoostingClassifier(n_estimators=100)\n",
    "clf7.fit(X_train, y_train)\n",
    "clf7_predictions = clf7.predict(X_test)\n",
    "clf7_accuracy = accuracy_score(y_test, clf7_predictions)\n",
    "clf7_f1 = f1_score(y_test, clf7_predictions)\n",
    "print(\"Gradient Boosting Classifier: [Accuracy: {:.4f}, f1-score: {:.4f}]\".format(clf7_accuracy, clf7_f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We optimize the above models with GridSearch using TimeSeriesSplit as cross-validation function\n",
    "\"\"\"\n",
    "For this next part, since it is very time consuming to actually do a GridSearch I have only put down the general codes used\n",
    "to optimize the algorithms. We will not see improved scores.\n",
    "\"\"\"\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import TimeSeriesSplit\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import fbeta_score, make_scorer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"Optimization of SVC\"\"\"\n",
    "# Choose the time series cross-validator\n",
    "tscv = TimeSeriesSplit(n_splits=10)\n",
    "# Choose SVC as the algorithm for optimization with GridSearch\n",
    "clf8 = svm.SVC(random_state = 0)\n",
    "# Create a dictionary for the parameters\n",
    "parameters_SVC = {'kernel':('linear', 'rbf', 'poly'), 'C':[0.001, 0.01, 0.1], 'degree':[200, 400, 600, 800]}\n",
    "# Define a scoring function\n",
    "scorer = make_scorer(f1_score)\n",
    "# Create the GridSearch object\"\"\"\n",
    "grid_obj_SVC = GridSearchCV(estimator=clf8, param_grid=parameters_SVC, scoring=scorer, cv=tscv)\n",
    "# Fit the grid search object to the data to compute the optimal model\n",
    "grid_fit_SVC = grid_obj_SVC.fit(X_train, y_train)\n",
    "# Return the optimal model after fitting the data\n",
    "best_clf_SVC = grid_fit_SVC.best_estimator_\n",
    "# Make predictions with the optimal model\n",
    "best_predictions_SVC = best_clf_SVC.predict(X_test)\n",
    "# Get the accuracy and f1_score of the optimized model\n",
    "clf8_optimized_accuracy = accuracy_score(y_test, best_predictions_SVC)\n",
    "clf8_optimized_f1 = f1_score(y_test, best_predictions_SVC)\n",
    "print(\"SVC Optimized [Accuracy score: {:.4f}, f1-score: {:.4f}]\".format(clf8_optimized_accuracy, clf8_optimized_f1))\n",
    "\n",
    "\"\"\"Optimization of decision tree\"\"\"\n",
    "# Choose DT as the algorithm for optimization with GridSearch\n",
    "clf9 = tree.DecisionTreeClassifier()\n",
    "# Create a dictionary for the parameters\n",
    "parameters_DT = {'criterion':('gini', 'entropy')}\n",
    "# Define a scoring function\n",
    "scorer = make_scorer(f1_score)\n",
    "# Create the GridSearch object\"\"\"\n",
    "grid_obj_DT = GridSearchCV(estimator=clf9, param_grid=parameters_DT, scoring=scorer, cv=tscv)\n",
    "# Fit the grid search object to the data to compute the optimal model\n",
    "grid_fit_DT = grid_obj_DT.fit(X_train, y_train)\n",
    "# Return the optimal model after fitting the data\n",
    "best_clf_DT = grid_fit_DT.best_estimator_\n",
    "# Make predictions with the optimal model\n",
    "best_predictions_DT = best_clf_DT.predict(X_test)\n",
    "# Get the accuracy and f1_score of the optimized model\n",
    "clf9_optimized_accuracy = accuracy_score(y_test, best_predictions_DT)\n",
    "clf9_optimized_f1 = f1_score(y_test, best_predictions_DT)\n",
    "print(\"Decidion Tree Optimized [Accuracy score: {:.4f}, f1-score: {:.4f}]\".format(clf9_optimized_accuracy, clf9_optimized_f1))\n",
    "\n",
    "\"\"\"Optimization of random forest\"\"\"\n",
    "# Choose RF as the algorithm for optimization with GridSearch\n",
    "clf10 = RandomForestClassifier()\n",
    "# Create a dictionary for the parameters\n",
    "parameters_RF = {'n_estimators':[5, 10, 20], 'criterion':('gini', 'entropy')}\n",
    "# Define a scoring function\n",
    "scorer = make_scorer(f1_score)\n",
    "# Create the GridSearch object\"\"\"\n",
    "grid_obj_RF = GridSearchCV(estimator=clf10, param_grid=parameters_RF, scoring=scorer, cv=tscv)\n",
    "# Fit the grid search object to the data to compute the optimal model\n",
    "grid_fit_RF = grid_obj_RF.fit(X_train, y_train)\n",
    "# Return the optimal model after fitting the data\n",
    "best_clf_RF = grid_fit_RF.best_estimator_\n",
    "# Make predictions with the optimal model\n",
    "best_predictions_RF = best_clf_RF.predict(X_test)\n",
    "# Get the accuracy and f1_score of the optimized model\n",
    "clf10_optimized_accuracy = accuracy_score(y_test, best_predictions_RF)\n",
    "clf10_optimized_f1 = f1_score(y_test, best_predictions_RF)\n",
    "print(\"Decidion Tree Optimized [Accuracy score: {:.4f}, f1-score: {:.4f}]\".format(clf10_optimized_accuracy, clf10_optimized_f1))\n",
    "\n",
    "\"\"\"Optimization of kNN\"\"\"\n",
    "# Choose the time series cross-validator\n",
    "tscv = TimeSeriesSplit(n_splits=3)\n",
    "# Choose kNN as the algorithm for optimization with GridSearch\n",
    "clf11 = KNeighborsClassifier()\n",
    "# Create a dictionary for the parameters\n",
    "parameters_kNN = {'weights':('uniform', 'distance'), 'n_neighbors':[3, 5, 7, 10, 25, 50, 100]}\n",
    "# Define a scoring function\n",
    "scorer = make_scorer(f1_score)\n",
    "# Create the GridSearch object\"\"\"\n",
    "grid_obj_kNN = GridSearchCV(estimator=clf11, param_grid=parameters_kNN, scoring=scorer, cv=tscv)\n",
    "# Fit the grid search object to the data to compute the optimal model\n",
    "grid_fit_kNN = grid_obj_kNN.fit(X_train, y_train)\n",
    "# Return the optimal model after fitting the data\n",
    "best_clf_kNN = grid_fit_kNN.best_estimator_\n",
    "# Make predictions with the optimal model\n",
    "best_predictions_kNN = best_clf_kNN.predict(X_test)\n",
    "# Get the accuracy and f1_score of the optimized model\n",
    "clf11_optimized_accuracy = accuracy_score(y_test, best_predictions_kNN)\n",
    "clf11_optimized_f1 = f1_score(y_test, best_predictions_kNN)\n",
    "print(\"kNN Optimized [Accuracy score: {:.4f}, f1-score: {:.4f}]\".format(clf11_optimized_accuracy, clf11_optimized_f1))\n",
    "\n",
    "\"\"\"Optimization of AdaBoost with DecisionTree\"\"\"\n",
    "# Choose the time series cross-validator\n",
    "tscv = TimeSeriesSplit(n_splits=3)\n",
    "# Choose AdaBoost as the algorithm for optimization with GridSearch\n",
    "clf12 = AdaBoostClassifier(base_estimator = DecisionTreeClassifier())\n",
    "# Create a dictionary for the parameters\n",
    "parameters_AdaBoost1 = {'n_estimators':[3, 5, 7, 10, 25, 50, 75, 100], 'learning_rate':[0.01, 0.05, 0.1, 0.5, 1]}\n",
    "# Define a scoring function\n",
    "scorer = make_scorer(f1_score)\n",
    "# Create the GridSearch object\"\"\"\n",
    "grid_obj_AdaBoost1 = GridSearchCV(estimator=clf12, param_grid=parameters_AdaBoost1, scoring=scorer, cv=tscv)\n",
    "# Fit the grid search object to the data to compute the optimal model\n",
    "grid_fit_AdaBoost1 = grid_obj_AdaBoost1.fit(X_train, y_train)\n",
    "# Return the optimal model after fitting the data\n",
    "best_clf_AdaBoost1 = grid_fit_AdaBoost1.best_estimator_\n",
    "# Make predictions with the optimal model\n",
    "best_predictions_AdaBoost1 = best_clf_AdaBoost1.predict(X_test)\n",
    "# Get the accuracy and f1_score of the optimized model\n",
    "clf12_optimized_accuracy = accuracy_score(y_test, best_predictions_AdaBoost1)\n",
    "clf12_optimized_f1 = f1_score(y_test, best_predictions_AdaBoost1)\n",
    "print(\"AdaBoost DT Optimized [Accuracy score: {:.4f}, f1-score: {:.4f}]\".format(clf12_optimized_accuracy, clf12_optimized_f1))\n",
    "\n",
    "\"\"\"Optimization of AdaBoost with GaussianNB\"\"\"\n",
    "# Choose the time series cross-validator\n",
    "tscv = TimeSeriesSplit(n_splits=3)\n",
    "# Choose AdaBoost as the algorithm for optimization with GridSearch\n",
    "clf13 = AdaBoostClassifier(base_estimator = GaussianNB())\n",
    "# Create a dictionary for the parameters\n",
    "parameters_AdaBoost2 = {'n_estimators':[3, 5, 7, 10, 25, 50, 75, 100], 'learning_rate':[0.01, 0.05, 0.1, 0.5, 1]}\n",
    "# Define a scoring function\n",
    "scorer = make_scorer(f1_score)\n",
    "# Create the GridSearch object\"\"\"\n",
    "grid_obj_AdaBoost2 = GridSearchCV(estimator=clf13, param_grid=parameters_AdaBoost2, scoring=scorer, cv=tscv)\n",
    "# Fit the grid search object to the data to compute the optimal model\n",
    "grid_fit_AdaBoost2 = grid_obj_AdaBoost2.fit(X_train, y_train)\n",
    "# Return the optimal model after fitting the data\n",
    "best_clf_AdaBoost2 = grid_fit_AdaBoost2.best_estimator_\n",
    "# Make predictions with the optimal model\n",
    "best_predictions_AdaBoost2 = best_clf_AdaBoost2.predict(X_test)\n",
    "# Get the accuracy and f1_score of the optimized model\n",
    "clf13_optimized_accuracy = accuracy_score(y_test, best_predictions_AdaBoost2)\n",
    "clf13_optimized_f1 = f1_score(y_test, best_predictions_AdaBoost2)\n",
    "print(\"AdaBoost DT Optimized [Accuracy score: {:.4f}, f1-score: {:.4f}]\".format(clf13_optimized_accuracy, clf13_optimized_f1))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}