1. k_fold_time_series_hyperoptimization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.014282,
     "end_time": "2023-06-03T01:46:16.695936",
     "exception": false,
     "start_time": "2023-06-03T01:46:16.681654",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# 1. Hyperparameter optimization \n",
    "To find the best hyperparemeter values, time-segmented k-fold evaluation is performed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "_kg_hide-input": false,
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2023-09-24T01:16:30.479757Z",
     "iopub.status.busy": "2023-09-24T01:16:30.479022Z",
     "iopub.status.idle": "2023-09-24T01:16:30.898036Z",
     "shell.execute_reply": "2023-09-24T01:16:30.896938Z",
     "shell.execute_reply.started": "2023-09-24T01:16:30.479686Z"
    },
    "papermill": {
     "duration": 4.795021,
     "end_time": "2023-06-03T01:46:36.791162",
     "exception": false,
     "start_time": "2023-06-03T01:46:31.996141",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#### Loading the libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.pipeline import Pipeline\n",
    "import matplotlib.pyplot as plt\n",
    "import xgboost as xgb\n",
    "from sklearn.impute import KNNImputer\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_folds(number_of_folds,rolling_test_size_ratio,fold_size,df):\n",
    "    \"\"\"\n",
    "    Splits a DataFrame into training and testing sets for cross-validation with a rolling window approach.\n",
    "    \n",
    "    Parameters:\n",
    "    - number_of_folds: The number of folds or partitions to create for cross-validation.\n",
    "    - rolling_test_size_ratio: The ratio of each fold to be used as the test set.\n",
    "    - fold_size: The size of each fold in terms of the DataFrame's rows.\n",
    "    - df: The DataFrame to be split into folds.\n",
    "\n",
    "    Returns:\n",
    "    - fold_dict: A dictionary containing each fold's DataFrame.\n",
    "    - X_train_dict, y_train_dict: Dictionaries containing the training data features and target values for each fold.\n",
    "    - X_test_dict, y_test_dict: Dictionaries containing the testing data features and target values for each fold.\n",
    "    \"\"\"\n",
    "    rolling_test_size = int(fold_size*rolling_test_size_ratio)\n",
    "    print(\"Creating\",number_of_folds,'folds')\n",
    "    # Preparing dictionaries\n",
    "    fold_dict = {}\n",
    "    X_train_dict = {}\n",
    "    y_train_dict = {}\n",
    "    X_test_dict = {}\n",
    "    y_test_dict = {}\n",
    "    for f_i in range(1,number_of_folds+1):\n",
    "        train_start_date = (f_i-1) * fold_size\n",
    "        train_end_date = f_i * fold_size - rolling_test_size\n",
    "        test_start_date = f_i * fold_size - rolling_test_size\n",
    "        test_end_date =  f_i * fold_size\n",
    "        \n",
    "        # Filtering folds from the dataframe\n",
    "        fold = df.loc[(df['date_id'] < test_end_date)& (df['date_id']  >= train_start_date)]\n",
    "        X_train = df.loc[(df['date_id'] < train_end_date)& (df['date_id']  >= train_start_date)][X_columns]\n",
    "        y_train = df.loc[(df['date_id'] < train_end_date)& (df['date_id']  >= train_start_date)][['target']]\n",
    "        \n",
    "        X_test =  df.loc[(df['date_id'] < test_end_date)& (df['date_id']  >=  test_start_date )][X_columns]\n",
    "        y_test = df.loc[(df['date_id'] < test_end_date)& (df['date_id']   >=  test_start_date )][['target']]\n",
    "        if f_i == number_of_folds: # Addressing the last fold\n",
    "            fold = df.loc[(df['date_id'] <= test_end_date)& (df['date_id']  >= train_start_date)]\n",
    "            X_test =  df.loc[(df['date_id'] <= test_end_date)& (df['date_id']  >=  test_start_date )][X_columns]\n",
    "            y_test = df.loc[(df['date_id'] <= test_end_date)& (df['date_id']   >=  test_start_date )][['target']]\n",
    "        \n",
    "        # Creating copies of the dataframes to avoid pointer issues\n",
    "        fold_dict[f_i] = fold.copy()\n",
    "        \n",
    "        X_train_dict[f_i] =  X_train.copy()\n",
    "        y_train_dict[f_i] =  y_train.copy()\n",
    "        \n",
    "        X_test_dict[f_i] =  X_test.copy()\n",
    "        y_test_dict[f_i] =  y_test.copy()\n",
    "    return fold_dict.copy(),X_train_dict.copy(), y_train_dict.copy(),X_test_dict.copy(), y_test_dict.copy()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_folds(df,fold_dict,X_train_dict, y_train_dict,X_test_dict, y_test_dict,par_dict):\n",
    "        \"\"\"\n",
    "        Evaluates model performance across all folds using Mean Absolute Error (MAE) as the metric.\n",
    "\n",
    "        Parameters:\n",
    "        - df: The original DataFrame.\n",
    "        - fold_dict: A dictionary of DataFrames representing each fold.\n",
    "        - X_train_dict, y_train_dict: Dictionaries containing the training features and target values for each fold.\n",
    "        - X_test_dict, y_test_dict: Dictionaries containing the testing features and target values for each fold.\n",
    "        - par_dict: A dictionary of model parameters.\n",
    "\n",
    "        Returns:\n",
    "        - mae_test_list: A list of MAE scores for the test sets across all folds.\n",
    "        - mae_train_list: A list of MAE scores for the training sets across all folds.\n",
    "        \"\"\"\n",
    "            mae_test_list = []\n",
    "            mae_train_list = []\n",
    "            for f_i in range(1,number_of_folds+1):\n",
    "                start_fold = time.time()\n",
    "                # Model is pretrained with the training data\n",
    "                model = xgb.XGBRegressor(base_score=0, booster='gbtree',    \n",
    "                           n_estimators=par_dict['n_estimators'],\n",
    "                           objective='reg:squarederror',\n",
    "                           max_depth=par_dict['max_depth'],\n",
    "                           eta=par_dict['eta'],\n",
    "                            min_child_weight=par_dict['min_child'],\n",
    "                            subsample =par_dict['subsample'],\n",
    "                            gamma =par_dict['gamma'],\n",
    "                            reg_lambda = par_dict['lambda'])\n",
    "                X_seconds_list = []\n",
    "                \n",
    "                # Setting up the dataframes\n",
    "                X_train_original = X_train_dict[f_i].copy()\n",
    "                y_train_original = y_train_dict[f_i].copy()\n",
    "                X_current_dataset = X_train_dict[f_i].copy()\n",
    "                y_current_dataset = y_train_dict[f_i].copy()\n",
    "                \n",
    "                # Training the model and giving predictions on the test set\n",
    "                model.fit(X_train_original , y_train_original)\n",
    "                y_train_prediction_results = model.predict(X_train_original)\n",
    "                X_train_original['target_pred'] = list(y_train_prediction_results)\n",
    "                X_train_original['target']  = y_train_original\n",
    "                X_train_original['absolute_error'] = abs(X_train_original['target'] - X_train_original['target_pred'])\n",
    "                mae_train_list.append(np.round(X_train_original['absolute_error'].mean(),5))\n",
    "                \n",
    "                for date_id in sorted(X_test_dict[f_i]['date_id'].unique()):\n",
    "                    # Retraining the model with previous day data\n",
    "                    test_start_date_id =  min(X_test_dict[f_i]['date_id'].unique())\n",
    "                    if date_id > test_start_date_id:\n",
    "                        X_previous_day_test = X_test_dict[f_i][X_test_dict[f_i]['date_id'] == date_id - 1].copy()\n",
    "                        y_previous_day_test =  y_test_dict[f_i][X_test_dict[f_i]['date_id'] == date_id - 1].copy()\n",
    "                        \n",
    "                        # Retraining the model\n",
    "                        X_current_dataset = pd.concat([ X_current_dataset,X_previous_day_test ]).reset_index(drop=True).copy()\n",
    "                        y_current_dataset = pd.concat([ y_current_dataset,y_previous_day_test]).reset_index(drop=True).copy()\n",
    "                        model.fit(X_current_dataset , y_current_dataset, xgb_model =  model.get_booster())    \n",
    "                        \n",
    "                    #Predicting the current day data\n",
    "                    X_day_test = X_test_dict[f_i][X_test_dict[f_i]['date_id'] == date_id].copy()\n",
    "                    y_day_test = y_test_dict[f_i][X_test_dict[f_i]['date_id'] == date_id].copy()\n",
    "                    \n",
    "                    # Predicting for each batch of 10 seconds\n",
    "                    for seconds_in_bucket in sorted(list(X_day_test['seconds_in_bucket'].unique())):\n",
    "                        X_seconds_test = X_day_test[X_day_test['seconds_in_bucket'] == seconds_in_bucket].copy()\n",
    "                        y_seconds_test = y_day_test[X_day_test['seconds_in_bucket'] == seconds_in_bucket].copy()\n",
    "  \n",
    "                        # Testing predictions\n",
    "                        X_seconds_test['target_pred'] = list(model.predict(X_seconds_test))\n",
    "                        X_seconds_test['target'] = y_seconds_test.copy()\n",
    "                        X_seconds_list.append(X_seconds_test.copy())\n",
    "                \n",
    "                    end_fold = time.time()\n",
    "                    print('Date:',date_id,'from',min( X_test_dict[f_i]['date_id'].unique()),'to', max(X_test_dict[f_i]['date_id'].unique()),'| Total time spent on this fold',np.round((end_fold - start_fold)/60,2),'minutes ****')                \n",
    "                X_test_df = pd.concat(X_seconds_list).copy()\n",
    "                X_test_df['absolute_error'] = abs(X_test_df['target'] - X_test_df['target_pred'])\n",
    "                mae_test_list.append(np.round(X_test_df['absolute_error'].mean(),5))\n",
    "            return  mae_test_list.copy(), mae_train_list.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### A bit of data cleaning to replace the missing nan values\n",
    "df['far_price'] = df.apply(\n",
    "    lambda row: row['reference_price'] if np.isnan(row['far_price']) else row['far_price'],\n",
    "    axis=1)\n",
    "\n",
    "df['near_price'] = df.apply(\n",
    "    lambda row: row['reference_price'] if np.isnan(row['near_price']) else row['near_price'],\n",
    "    axis=1)\n",
    "\n",
    "### Droping rows with missing values\n",
    "df = df.dropna()\n",
    "\n",
    "### Print statement to indicate the start of fold creation\n",
    "print(\"Creating folds\")\n",
    "\n",
    "### Constants for fold creation\n",
    "number_of_stocks = 200\n",
    "number_of_bucket_iter = 55\n",
    "number_of_folds = 3\n",
    "total_number_of_days = 480\n",
    "fold_size =  total_number_of_days/number_of_folds\n",
    "rolling_test_size_ratio = 0.01\n",
    "rolling_test_size = int(fold_size*rolling_test_size_ratio)\n",
    "\n",
    "### Hyperparameter lists for tuning\n",
    "max_depth_list = [1,2,3,4,5]\n",
    "n_estimators_list = [3,10]\n",
    "gamma_list = [0,100]\n",
    "lambda_list = [0,100]\n",
    "min_child_list = [0,100]\n",
    "subsample_list = [1,0.5]\n",
    "eta_list = [0.3,0.001]\n",
    "\n",
    "### Total number of iterations for hyperparameter tuning\n",
    "total_iter = len(max_depth_list) * len(n_estimators_list) * len(gamma_list) * len(min_child_list)*len(subsample_list)*len(eta_list)*len(lambda_list)\n",
    "\n",
    "### Dictionaries for storing results\n",
    "result_mean_dict = {}\n",
    "result_std_dict = {}\n",
    "results_df_dict =  {'max_depth':[],'n_estimators':[],'eta':[],'min_child':[],'subsample':[],'gamma':[],'lambda':[],'train_mean_mae':[],'train_std_mae':[],'test_mean_mae':[],'test_std_mae':[]}\n",
    "count = 1\n",
    "\n",
    "### Printing statement to indicate the start of hyperparameter evaluation\n",
    "print('Hyperparameter evaluation starting')\n",
    "### Dictionary to hold method-specific configurations\n",
    "method_dict = {'model_name' : 'all_stocks_xgboost',\n",
    "'retraining_freq' : 'daily_retraining',\n",
    "'retraining_method' : 'on_full_data'}\n",
    "\n",
    "### Looping through all combinations of hyperparameters\n",
    "for subsample in subsample_list:\n",
    "    for min_child in min_child_list:\n",
    "        for gamma in gamma_list:\n",
    "            for lambda_p in lambda_list:\n",
    "                for eta in eta_list:\n",
    "                    for n_estimators in n_estimators_list:\n",
    "                        for max_depth in max_depth_list:\n",
    "                            start = time.time()\n",
    "                            ### Dictionary for current set of hyperparameters\n",
    "                            par_dict = {}\n",
    "                            par_dict['n_estimators'] = n_estimators\n",
    "                            par_dict['max_depth'] = max_depth\n",
    "                            par_dict['gamma'] = gamma\n",
    "                            par_dict['subsample'] = subsample\n",
    "                            par_dict['min_child'] = min_child\n",
    "                            par_dict['eta'] = eta\n",
    "                            par_dict['lambda'] = lambda_p\n",
    "\n",
    "                            ### Appending current hyperparameters to results dictionary\n",
    "                            results_df_dict['n_estimators'].append(n_estimators)        \n",
    "                            results_df_dict['max_depth'].append(max_depth)\n",
    "                            results_df_dict['gamma'].append(gamma)\n",
    "                            results_df_dict['subsample'].append(subsample)\n",
    "                            results_df_dict['min_child'].append(min_child)\n",
    "                            results_df_dict['eta'].append\n",
    "                            results_df_dict['lambda'].append(lambda_p) \n",
    "                            \n",
    "                            ### Splitting the folds\n",
    "                            fold_dict,X_train_dict, y_train_dict,X_test_dict, y_test_dict = create_folds(number_of_folds,rolling_test_size_ratio,fold_size,df.copy())\n",
    "                            \n",
    "                            ### Obtaining test results and storing them\n",
    "                            mae_test_list, mae_train_list = evaluate_folds(df.copy(),fold_dict.copy(),X_train_dict.copy(), y_train_dict.copy(),X_test_dict.copy(), y_test_dict.copy(),par_dict.copy())\n",
    "\n",
    "                            test_mean = np.round(np.mean(mae_test_list.copy()),5)\n",
    "                            test_std = np.round(np.std(mae_test_list.copy()),5)\n",
    "                            results_df_dict['test_mean_mae'].append(test_mean)\n",
    "                            results_df_dict['test_std_mae'].append(test_std)\n",
    "\n",
    "                            train_mean = np.round(np.mean(mae_train_list.copy()),5)\n",
    "                            train_std = np.round(np.std(mae_train_list.copy()),5)        \n",
    "                            results_df_dict['train_mean_mae'].append(train_mean)\n",
    "                            results_df_dict['train_std_mae'].append(train_std)\n",
    "\n",
    "                            result_df = pd.DataFrame(results_df_dict)\n",
    "                            result_df.to_csv(method_dict['model_name'] + '_' + method_dict['retraining_freq'] + '_' + method_dict['retraining_method']+ '_n_folds_' + str(number_of_folds) + '_test_size_' + str(rolling_test_size) + '.csv')\n",
    "                            # Print progress update and timing\n",
    "                            end = time.time()\n",
    "                            print('**** Validation',count,'out of',total_iter,'Total time spent on these hyperparameters',np.round((end - start)/60,2),'minutes ****')\n",
    "                            count+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ranking the test set results\n",
    "result_df.sort_values(['test_mean_mae'], ascending=[True]).round(4)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}