Untitled1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using Arima on the Tronderenergi dataset, 24 hours multistep forecast\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from statsmodels.tsa.arima_model import ARIMA\n",
    "from data_extraction import *\n",
    "import warnings\n",
    "import numpy as np\n",
    "warnings.filterwarnings('ignore')\n",
    "from datetime import datetime\n",
    "from datetime import timedelta\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "name_of_data = 'train' #choose the appropriate data (train, test or test_backfilled_missing_data)\n",
    "n_rows_read = None # choose number of rows (use None to display all rows)\n",
    "grid_number = 1 # choose grid number (current dataset has data from 3 grids so you can choose between 1,2 or 3)\n",
    "wanted_feature = 'loss' # choose name of feature (could be load, loss, temp, or predictive values)\n",
    "feature_unit = 'MWh' # change to Kelvin if the feature is temp (temperature)\n",
    "df = get_timeseries(name_of_data,n_rows_read,wanted_feature,1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(df)\n",
    "df = df.rename(columns={'index':'timestamp',f'grid{grid_number}-loss':'grid-loss'})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def difference(df,interval,difference_column):\n",
    "    temp = df.copy()\n",
    "    temp['shifted'] = temp[difference_column].shift(interval)\n",
    "    temp['differenced'] = temp[difference_column]-temp['shifted']\n",
    "    return temp[['differenced']]\n",
    "\n",
    "def invert_difference(df, forecast_df, difference_interval, difference_column):\n",
    "    temp = df.copy()\n",
    "    temp['shifted'] = temp[difference_column].shift(difference_interval)\n",
    "    return forecast_df + temp['shifted']\n",
    "\n",
    "def ARIMA_train(data,order):\n",
    "    train = df.copy()\n",
    "    # Differencing train set\n",
    "    # REMARK:dropping the values from 0-differnce_interval\n",
    "    train['first_difference'] = difference(train, 365*24, 'grid-loss')\n",
    "    train['second_difference'] = difference(train, 24, 'first_difference')\n",
    "    train = train.dropna()\n",
    "    # Construct model\n",
    "    model = ARIMA(train[['second_difference']], order=order)\n",
    "    fitted = model.fit(disp=0)\n",
    "    \n",
    "    return fitted\n",
    "\n",
    "    \n",
    "\n",
    "def ARIMA_invert_result(data,fitted_model):\n",
    "    #INPUT: original data and fitted model with second degree difference\n",
    "    #OUTPUT: DataFrame with columns: grid-loss, ARIMA, and residual\n",
    "    temp = df.copy()\n",
    "    #Differencing original data\n",
    "    temp['year_difference'] = difference(temp, 24*365, 'grid-loss')\n",
    "    temp['day_difference'] = difference(temp, 24, 'year_difference')\n",
    "    #Extract results from second degree differed ARIMA model on the training data\n",
    "    temp['ARIMA'] = fitted_model.fittedvalues\n",
    "    #Inverts the result\n",
    "    temp['day_inverted'] = invert_difference(temp, temp['ARIMA'], 24, 'year_difference')\n",
    "    temp['ARIMA'] = invert_difference(temp, temp['day_inverted'], 24*365, 'grid-loss')\n",
    "    #Calculate residual\n",
    "    temp['residual'] = temp['grid-loss']-temp['ARIMA']\n",
    "    return temp[['grid-loss','ARIMA','residual']].dropna()  \n",
    "\n",
    "def to_datetime(string):\n",
    "    return datetime.strptime(string, \"%Y-%m-%d %H:%M:%S\")\n",
    "def to_string(date):\n",
    "    return datetime.strftime(date, \"%Y-%m-%d %H:%M:%S\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "##Hyperparameters\n",
    "order = (9,2,7)\n",
    "######### DATA EXTRACTION ####################\n",
    "# Extracting train data\n",
    "train = get_timeseries('train',None,'loss',1)\n",
    "train = pd.DataFrame(train)\n",
    "train = train.rename(columns={'index':'timestamp',f'grid{grid_number}-loss':'grid-loss'})\n",
    "# Extracting test data\n",
    "test = get_timeseries('test',None,'loss',1)\n",
    "test = pd.DataFrame(test)\n",
    "test = test.rename(columns={'index':'timestamp',f'grid{grid_number}-loss':'grid-loss'})\n",
    "\n",
    "############ Training ##########################\n",
    "#fitted_model = ARIMA_train(df = train,order=(9,0,7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STATUS:2020-05-30 00:00:00/2020-05-31 00:00:00 --- Last Duration:0:00:00\r"
     ]
    }
   ],
   "source": [
    "def to_datetime(string):\n",
    "    return datetime.strptime(string, \"%Y-%m-%d %H:%M:%S\")\n",
    "def to_string(date):\n",
    "    return datetime.strftime(date, \"%Y-%m-%d %H:%M:%S\")\n",
    "\n",
    "def ARIMA_24_hour_forecast(df,forecast_start, order):\n",
    "    temp = df.copy()\n",
    "    #Differencing original data\n",
    "    temp['year_difference'] = difference(temp, 24*365, 'grid-loss')\n",
    "    temp['day_difference'] = difference(temp, 24, 'year_difference')\n",
    "    fitted_model = ARIMA_train(temp[:to_string(forecast_start)],order)\n",
    "    \n",
    "    forecast = fitted_model.forecast(steps=24)[0]\n",
    "    #Calculate new index\n",
    "    new_date_index = [to_string(forecast_start + timedelta(hours=i)) for i in range(1,24+1)]\n",
    "    \n",
    "    forecast_df = pd.DataFrame(index = new_date_index)\n",
    "    forecast_df['ARIMA_differenced'] = forecast\n",
    "    \n",
    "    #Use yearlier values to invert difference\n",
    "    temp = pd.concat([temp,forecast_df])\n",
    "    temp['day_inverted'] = invert_difference(temp, temp['ARIMA_differenced'], 24, 'year_difference')\n",
    "    temp['ARIMA'] = invert_difference(temp, temp['day_inverted'], 24*365, 'grid-loss')\n",
    "    \n",
    "    #Extracts only the 24 hour forecast\n",
    "    forecast_df = temp['ARIMA'].dropna()\n",
    "    \n",
    "    return pd.DataFrame(forecast_df)\n",
    " \n",
    "\n",
    "def daterange(start_date, end_date):\n",
    "    for n in range(int((end_date - start_date).days)):\n",
    "        yield start_date + timedelta(n)\n",
    "\n",
    "\n",
    "temp = pd.concat([train,test])\n",
    "forecast_df = pd.DataFrame()\n",
    "min_train_size = 365*24 + 24 + 24*7\n",
    "\n",
    "#Calculate start date\n",
    "start_date = to_datetime(temp.index[min_train_size])-timedelta(hours=1)\n",
    "#Calculate end date\n",
    "end_date = to_datetime(temp.index[-1])\n",
    "end_date = datetime(end_date.year,end_date.month,end_date.day)\n",
    "elapsed_time = 0\n",
    "time_df = pd.DataFrame()\n",
    "for forecast_date in daterange(start_date,end_date):\n",
    "    #print(f\"Last duration: {elapsed_time}\",end='\\r')\n",
    "    print(f\"STATUS:{to_string(forecast_date)}/{to_string(end_date)} --- Last Duration:{elapsed_time}\",end='\\r')\n",
    "    t1 = time.time()\n",
    "    forecast = ARIMA_24_hour_forecast(temp[:to_string(forecast_date)], forecast_date,(1,0,0))\n",
    "    forecast_df = forecast_df.append(forecast)\n",
    "    forecast_date += timedelta(days=1)\n",
    "    #Calculate duration\n",
    "    t2 = time.time()\n",
    "    h, rem = divmod(t2-t1, 3600)\n",
    "    m, s = divmod(rem, 60)\n",
    "    elapsed_time = timedelta(hours=int(h), minutes=int(m), seconds = int(s))\n",
    "    row = pd.DataFrame([elapsed_time], index = [forecast_date], columns = ['Computation time'])\n",
    "    time_df = time_df.append(row)\n",
    "    time_df.to_pickle('computation.pkl')\n",
    "\n",
    "temp = temp.join(forecast_df)\n",
    "temp = temp.dropna()\n",
    "temp.to_pickle('forecast.pkl')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>grid-loss</th>\n",
       "      <th>ARIMA</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2018-12-09 01:00:00</th>\n",
       "      <td>18.50200</td>\n",
       "      <td>18.810722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-12-09 02:00:00</th>\n",
       "      <td>17.90600</td>\n",
       "      <td>18.039181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-12-09 03:00:00</th>\n",
       "      <td>17.57600</td>\n",
       "      <td>17.771552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-12-09 04:00:00</th>\n",
       "      <td>16.82600</td>\n",
       "      <td>18.001004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-12-09 05:00:00</th>\n",
       "      <td>17.89300</td>\n",
       "      <td>18.230697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-05-30 19:00:00</th>\n",
       "      <td>8.55980</td>\n",
       "      <td>7.129942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-05-30 20:00:00</th>\n",
       "      <td>6.95209</td>\n",
       "      <td>6.751133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-05-30 21:00:00</th>\n",
       "      <td>8.11558</td>\n",
       "      <td>8.986810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-05-30 23:00:00</th>\n",
       "      <td>7.26431</td>\n",
       "      <td>11.185511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2020-05-31 00:00:00</th>\n",
       "      <td>6.78796</td>\n",
       "      <td>8.928043</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>12902 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     grid-loss      ARIMA\n",
       "2018-12-09 01:00:00   18.50200  18.810722\n",
       "2018-12-09 02:00:00   17.90600  18.039181\n",
       "2018-12-09 03:00:00   17.57600  17.771552\n",
       "2018-12-09 04:00:00   16.82600  18.001004\n",
       "2018-12-09 05:00:00   17.89300  18.230697\n",
       "...                        ...        ...\n",
       "2020-05-30 19:00:00    8.55980   7.129942\n",
       "2020-05-30 20:00:00    6.95209   6.751133\n",
       "2020-05-30 21:00:00    8.11558   8.986810\n",
       "2020-05-30 23:00:00    7.26431  11.185511\n",
       "2020-05-31 00:00:00    6.78796   8.928043\n",
       "\n",
       "[12902 rows x 2 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_pickle('forecast.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>from</th>\n",
       "      <th>to</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0 days 02:00:00</td>\n",
       "      <td>2020-05-30 21:00:00</td>\n",
       "      <td>2020-05-30 23:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1 days 01:00:00</td>\n",
       "      <td>2020-01-15 22:00:00</td>\n",
       "      <td>2020-01-16 23:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0 days 04:00:00</td>\n",
       "      <td>2019-12-26 22:00:00</td>\n",
       "      <td>2019-12-27 02:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0 days 07:00:00</td>\n",
       "      <td>2019-12-25 22:00:00</td>\n",
       "      <td>2019-12-26 05:00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         duration                 from                   to\n",
       "3 0 days 02:00:00  2020-05-30 21:00:00  2020-05-30 23:00:00\n",
       "2 1 days 01:00:00  2020-01-15 22:00:00  2020-01-16 23:00:00\n",
       "1 0 days 04:00:00  2019-12-26 22:00:00  2019-12-27 02:00:00\n",
       "0 0 days 07:00:00  2019-12-25 22:00:00  2019-12-26 05:00:00"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "########FIND GAPS IN DATASET###########\n",
    "temp = pd.read_pickle('forecast.pkl')\n",
    "time = temp.index\n",
    "d = {}\n",
    "d['duration'] = []\n",
    "d['from'] = []\n",
    "d['to'] = []\n",
    "for i in range(len(time)-1):\n",
    "    delta = to_datetime(time[i+1])-to_datetime(time[i])\n",
    "    if delta > timedelta(hours=1):\n",
    "        d['duration'].append(delta)\n",
    "        d['from'].append(time[i])\n",
    "        d['to'].append(time[i+1])\n",
    "\n",
    "gaps = pd.DataFrame(d)\n",
    "gaps.sort_values(by=['from'],ascending=False)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[datetime.timedelta(seconds=7200),\n",
       " datetime.timedelta(seconds=25200),\n",
       " datetime.timedelta(seconds=14400),\n",
       " datetime.timedelta(days=1, seconds=3600),\n",
       " datetime.timedelta(seconds=7200)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}