meta_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "according-beauty",
   "metadata": {},
   "outputs": [],
   "source": [
    "import utils\n",
    "import math\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "from tqdm import tqdm\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "executed-burlington",
   "metadata": {},
   "source": [
    "# Train Meta Models for All"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "nonprofit-ground",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define variables and paths\n",
    "path = 'meta_model_predictions/'\n",
    "model_paths = {'logreg': 'categorical/logreg/pct_macro_thresh', \\\n",
    "               'rf': 'categorical/rf/pct_tech_macro', \\\n",
    "               'xgb': 'categorical/xgb/pct_tech', \\\n",
    "               'arima': 'arimaEnsemble', \\\n",
    "               # 'emaStrategy': 'emaStrategy', \\\n",
    "               # 'fourCandleHammer': 'fourCandleHammer', \\\n",
    "               #'swingSetup': 'swingSetup'\n",
    "              }\n",
    "\n",
    "train_start = datetime(2018, 10, 1)\n",
    "train_end = datetime(2020, 9, 30)\n",
    "val_start = datetime(2020, 10, 1)\n",
    "val_end = datetime(2020, 12, 31)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "outstanding-demand",
   "metadata": {},
   "outputs": [],
   "source": [
    "xgbParams = [{'booster': ['gbtree'],\n",
    "              'learning_rate': [0.01, 0.1, 0.3], # default 0.3\n",
    "              'gamma': [0, 0.5, 1], # higher means more regularization\n",
    "              'max_depth': [2, 4, 6, 8], # default 6\n",
    "}]\n",
    "\n",
    "# parameter grid\n",
    "parameter_grid = list(ParameterGrid(xgbParams))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "detected-adams",
   "metadata": {},
   "outputs": [],
   "source": [
    "# store predictions in a dictionary\n",
    "future_preds_dict = dict()\n",
    "\n",
    "# store validation data in a dictionary\n",
    "# val_data = dict()\n",
    "\n",
    "for future in tqdm(utils.futuresList):\n",
    "    # print(future)\n",
    "    # merge base model probability predictions\n",
    "    future_df = pd.DataFrame()\n",
    "    for model_name, model_path in model_paths.items():\n",
    "        data = pd.read_csv(f'{path}{model_path}/{future}.csv')\n",
    "        data.columns = ['date', model_name]\n",
    "        data['date'] = pd.to_datetime(data['date'])\n",
    "        data = data.set_index('date')\n",
    "        # filter data within date range\n",
    "        data = data.loc[(data.index >= train_start) & (data.index <= val_end)]\n",
    "        if len(future_df) == 0:\n",
    "            future_df = data.copy()\n",
    "        else:\n",
    "            future_df = pd.merge(future_df, data, on=['date'], how='outer')\n",
    "\n",
    "    # load and extract y_variable data\n",
    "    y_var = \"LONG_SHORT\" # DO NOT CHANGE THIS\n",
    "    df = utils.prepare_data(future)\n",
    "    df = df.loc[(df.index >= train_start) & (df.index <= val_end)]\n",
    "    future_df[y_var] = df[y_var].values\n",
    "    \n",
    "    # train test split\n",
    "    train_df = future_df.loc[(future_df.index >= train_start) & (future_df.index <= train_end)]\n",
    "    val_df = future_df.loc[(future_df.index >= val_start) & (future_df.index <= val_end)]\n",
    "    # val_data[future] = val_df.copy() # store in validation dictionary\n",
    "    \n",
    "    X_train = train_df[model_paths.keys()]\n",
    "    y_train = train_df[y_var]\n",
    "    \n",
    "    X_val = val_df[model_paths.keys()]\n",
    "        \n",
    "    for i in range(len(parameter_grid)):\n",
    "        param_set = parameter_grid[i]\n",
    "        params = f\"lr{param_set['learning_rate']}_g{param_set['gamma']}_d{param_set['max_depth']}\"\n",
    "        xgb_model = XGBClassifier(objective='binary:logistic', eval_metric='logloss', **param_set)\n",
    "        \n",
    "        # train meta model\n",
    "        # xgb_model = XGBClassifier(objective='binary:logistic', eval_metric='logloss')\n",
    "        xgb_model.fit(X_train, y_train)\n",
    "\n",
    "        # save model\n",
    "        # with open(f'saved_models/meta/meta_model_base/{future}.p', 'wb') as f:\n",
    "        #     pickle.dump(xgb_model, f)\n",
    "\n",
    "        # load saved model\n",
    "        # with open(f'saved_models/meta/meta_model_base/{future}.p', 'rb') as f:\n",
    "        #     model = pickle.load(f)\n",
    "\n",
    "        # predict prob long\n",
    "        future_preds = xgb_model.predict_proba(X_val)[:, 1]\n",
    "        \n",
    "        future_preds_df = pd.DataFrame(index=X_val.index)\n",
    "        future_preds_df['preds'] = future_preds\n",
    "\n",
    "        # store predictions in a dictionary\n",
    "        future_preds_dict[future, params] = future_preds_df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "recovered-access",
   "metadata": {},
   "outputs": [],
   "source": [
    "# save dictionary\n",
    "with open(\"meta_model_predictions/future_preds_dict_3.pkl\", \"wb\") as f:\n",
    "    pickle.dump(future_preds_dict, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "considerable-yeast",
   "metadata": {},
   "outputs": [],
   "source": [
    "# saved_dict = pickle.load(open(\"meta_model_predictions/future_preds_dict_2.pkl\", \"rb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "palestinian-cornell",
   "metadata": {},
   "source": [
    "# Save Final Meta Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "offshore-consumption",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define variables and paths\n",
    "path = 'meta_model_predictions/'\n",
    "model_paths = {'logreg': 'categorical/logreg/pct_macro', \\\n",
    "               'rf': 'categorical/rf/pct_tech_macro', \\\n",
    "               'xgb': 'categorical/xgb/pct_tech', \\\n",
    "               'arima': 'arimaEnsemble', \\\n",
    "               'emaStrategy': 'emaStrategy', \\\n",
    "               'fourCandleHammer': 'fourCandleHammer', \\\n",
    "               'swingSetup': 'swingSetup'}\n",
    "\n",
    "train_start = datetime(2019, 1, 1)\n",
    "train_end = datetime(2020, 12, 31)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "forward-taiwan",
   "metadata": {},
   "outputs": [],
   "source": [
    "finalParams = [{'booster': ['gbtree'],\n",
    "              'learning_rate': [0.01], # default 0.3\n",
    "              'gamma': [1], # higher means more regularization\n",
    "              'max_depth': [2], # default 6\n",
    "}]\n",
    "\n",
    "# parameter grid\n",
    "parameter_grid = list(ParameterGrid(finalParams))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "recorded-electricity",
   "metadata": {},
   "outputs": [],
   "source": [
    "# store predictions in a dictionary\n",
    "future_preds_dict = dict()\n",
    "\n",
    "# store validation data in a dictionary\n",
    "# val_data = dict()\n",
    "\n",
    "for future in tqdm(utils.futuresList):\n",
    "    # print(future)\n",
    "    # merge base model probability predictions\n",
    "    future_df = pd.DataFrame()\n",
    "    for model_name, model_path in model_paths.items():\n",
    "        data = pd.read_csv(f'{path}{model_path}/{future}.csv')\n",
    "        data.columns = ['date', model_name]\n",
    "        data['date'] = pd.to_datetime(data['date'])\n",
    "        data = data.set_index('date')\n",
    "        # filter data within date range\n",
    "        data = data.loc[(data.index >= train_start) & (data.index <= train_end)]\n",
    "        if len(future_df) == 0:\n",
    "            future_df = data.copy()\n",
    "        else:\n",
    "            future_df = pd.merge(future_df, data, on=['date'], how='outer')\n",
    "\n",
    "    # load and extract y_variable data\n",
    "    y_var = \"LONG_SHORT\" # DO NOT CHANGE THIS\n",
    "    df = utils.prepare_data(future)\n",
    "    df = df.loc[(df.index >= train_start) & (df.index <= train_end)]\n",
    "    future_df[y_var] = df[y_var].values\n",
    "    \n",
    "    # train test split\n",
    "    train_df = future_df.loc[(future_df.index >= train_start) & (future_df.index <= train_end)]\n",
    "    \n",
    "    X_train = train_df[model_paths.keys()]\n",
    "    y_train = train_df[y_var]\n",
    "            \n",
    "    xgb_model = XGBClassifier(objective='binary:logistic', eval_metric='logloss', **parameter_grid[0])\n",
    "        \n",
    "    # train meta model\n",
    "    # xgb_model = XGBClassifier(objective='binary:logistic', eval_metric='logloss')\n",
    "    xgb_model.fit(X_train, y_train)\n",
    "\n",
    "    # save model\n",
    "    with open(f'saved_models/meta/meta_model_final_2/{future}.p', 'wb') as f:\n",
    "        pickle.dump(xgb_model, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "english-instrument",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "strategic-resident",
   "metadata": {},
   "source": [
    "# Retrieve Sharpe from Quantiacs\n",
    "Transferred to `meta_model_validation.py`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "verbal-centre",
   "metadata": {},
   "outputs": [],
   "source": [
    "import quantiacsToolbox\n",
    "\n",
    "from strategy import (\n",
    "    basic_strategy, \n",
    "    long_only,\n",
    "    short_only,\n",
    "    fixed_threshold_strategy, \n",
    "    perc_threshold_strategy,\n",
    "    futures_only,\n",
    "    futures_hold,\n",
    "    cash_and_futures,\n",
    ")\n",
    "\n",
    "# for i in range(len(parameter_grid)):\n",
    "i = 17 ## change this\n",
    "\n",
    "param_set = parameter_grid[i]\n",
    "\n",
    "params = f\"lr{param_set['learning_rate']}_g{param_set['gamma']}_d{param_set['max_depth']}\"\n",
    "\n",
    "# retrieve preds\n",
    "future_preds_dict = pickle.load(open(\"meta_model_predictions/future_preds_dict.pkl\", \"rb\"))\n",
    "preds_dict_final = dict()\n",
    "for future in utils.futuresList:\n",
    "    preds_dict_final[future] = future_preds_dict[future, params].copy()\n",
    "\n",
    "# define class\n",
    "class metaModelValidation(object):\n",
    "    # throw preds into quantiacs\n",
    "    def myTradingSystem(self, DATE, settings):\n",
    "        ''' This system uses trend following techniques to allocate capital into the desired equities'''\n",
    "\n",
    "        # Get saved X variables\n",
    "        prediction = pd.DataFrame(index=utils.futuresList)\n",
    "        for future in tqdm(utils.futuresList):\n",
    "            # read data\n",
    "            try:\n",
    "                future_pred = settings['saved_predictions'][future].loc[datetime.strptime(str(DATE[-1]), '%Y%m%d')][0]\n",
    "                prediction.loc[future, 'meta'] = future_pred\n",
    "            except:\n",
    "                print('ERROR: ', future, str(DATE[-1]))\n",
    "                prediction.loc[future, 'meta'] = 0\n",
    "\n",
    "        sign = utils.sign(prediction)\n",
    "        magnitude = utils.magnitude(prediction)\n",
    "\n",
    "        position = basic_strategy(sign['meta'], magnitude['meta']) \n",
    "\n",
    "        # Cash-futures strategy\n",
    "        position = futures_only(position)\n",
    "\n",
    "        # Update persistent data across runs\n",
    "        settings['sign'].append(sign)\n",
    "        settings['magnitude'].append(magnitude)\n",
    "        settings['previous_position'] = position\n",
    "\n",
    "        # Yay!\n",
    "        return position, settings\n",
    "\n",
    "    def mySettings(self):\n",
    "        ''' Define your trading system settings here '''\n",
    "        settings= {}\n",
    "        settings['markets']  = utils.futuresAllList\n",
    "        settings['beginInSample'] = '20181020'\n",
    "        settings['endInSample'] = '20201231'\n",
    "        settings['lookback']= 504\n",
    "        settings['budget']= 10**6\n",
    "        settings['slippage']= 0.05\n",
    "\n",
    "        # Stuff to persist\n",
    "        settings['saved_predictions'] = preds_dict_final ## update this\n",
    "        settings['sign'] = []\n",
    "        settings['magnitude'] = []\n",
    "\n",
    "        return settings\n",
    "\n",
    "results = quantiacsToolbox.runts(metaModelValidation, plotEquity=False)\n",
    "sharpe = results[\"stats\"][\"sharpe\"]\n",
    "\n",
    "sharpe_results = pd.read_csv(\"meta_model_predictions/sharpe_results.csv\")\n",
    "sharpe_results = sharpe_results.append({'params': params, 'sharpe': sharpe}, ignore_index=True)\n",
    "sharpe_results.to_csv(\"meta_model_predictions/sharpe_results.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "alert-berry",
   "metadata": {},
   "source": [
    "# Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dynamic-watch",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import utils\n",
    "\n",
    "best_futures = [\"F_LB\"]\n",
    "for future in best_futures:\n",
    "    print(future)\n",
    "    \n",
    "    with open(f'saved_models/meta/meta_model_final_2/{future}.p', 'rb') as f:\n",
    "        meta_wrapper = pickle.load(f)\n",
    "\n",
    "    X_vars = ['logreg', 'rf', 'xgb', 'arimaEnsemble', 'emaStrategy', 'fourCandleHammer', 'swingSetup']\n",
    "\n",
    "    for x in zip(X_vars, meta_wrapper.feature_importances_):\n",
    "        print(x[0], x[1])\n",
    "    \n",
    "    print(\"--------------------------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "forty-salad",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Edit Metadata",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}