group26_ie360.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import zipfile\n",
    "import os\n",
    "from io import BytesIO\n",
    "\n",
    "def download_and_extract_zip(zip_url, extract_path):\n",
    "    \"\"\"Downloads and extracts a ZIP file from Google Drive to the specified path.\"\"\"\n",
    "\n",
    "    try:\n",
    "        response = requests.get(zip_url)\n",
    "        response.raise_for_status()  # Raise an error for bad responses\n",
    "\n",
    "        with zipfile.ZipFile(BytesIO(response.content)) as zip_file:\n",
    "            for member in zip_file.namelist():\n",
    "                filename = os.path.basename(member)\n",
    "                if not filename:\n",
    "                    continue  # Skip directories\n",
    "\n",
    "                source = zip_file.open(member)\n",
    "                target_path = os.path.join(extract_path, filename)\n",
    "\n",
    "                with open(target_path, \"wb\") as target_file:\n",
    "                    target_file.write(source.read())\n",
    "\n",
    "    except requests.exceptions.RequestException as e:\n",
    "        print(f\"Download error for '{zip_url}': {e}\")\n",
    "    except zipfile.BadZipFile:\n",
    "        print(f\"Invalid ZIP file: '{zip_url}'.\")\n",
    "    except Exception as e:\n",
    "        print(f\"Unexpected error for '{zip_url}': {e}\")\n",
    "\n",
    "# List of Google Drive file IDs\n",
    "zip_file_ids = [\"1As-67MFNrpim_jGYgSOgT2SF119gP790\", \"1pjaRe_lSIygHyZ8luWplSC6fuwAEmwTd\"]\n",
    "\n",
    "# Get the current working directory\n",
    "script_directory = os.getcwd()\n",
    "\n",
    "# Construct direct download links for the ZIP files\n",
    "zip_urls = [f\"https://drive.google.com/uc?export=download&id={file_id}\" for file_id in zip_file_ids]\n",
    "\n",
    "# Download and extract each ZIP file directly into the current working directory\n",
    "for zip_url in zip_urls:\n",
    "    download_and_extract_zip(zip_url, script_directory)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import glob\n",
    "\n",
    "# Read CSV files\n",
    "production_data = pd.read_csv(\"production.csv\")\n",
    "production_data['date'] = pd.to_datetime(production_data['date'])\n",
    "\n",
    "weather_data = pd.read_csv(\"processed_weather.csv\")\n",
    "weather_data['date'] = pd.to_datetime(weather_data['date'])\n",
    "\n",
    "# Sort the data by date and hour\n",
    "production_data = production_data.sort_values(by=[\"date\", \"hour\"])\n",
    "weather_data = weather_data.sort_values(by=[\"date\", \"hour\", \"lat\", \"lon\"])\n",
    "\n",
    "# Fill missing values in weather_data using the previous day's same hour, lat, and lon values\n",
    "for col in weather_data.columns:\n",
    "    if col in ['date', 'hour', 'lat', 'lon']:\n",
    "        continue\n",
    "    weather_data[col] = weather_data.groupby(['hour', 'lat', 'lon'])[col].transform(lambda x: x.fillna(method='ffill'))\n",
    "\n",
    "# Identify common dates and hours present in both datasets\n",
    "common_dates = set(production_data[\"date\"].unique()) & set(weather_data[\"date\"].unique())\n",
    "common_hours = set(production_data[\"hour\"].unique()) & set(weather_data[\"hour\"].unique())\n",
    "\n",
    "# Save each coordinate's weather data to a separate file\n",
    "for lat in weather_data[\"lat\"].unique():\n",
    "    for lon in weather_data[\"lon\"].unique():\n",
    "        # Filter weather data for the specific coordinate\n",
    "        coord_weather = weather_data[(weather_data[\"lat\"] == lat) & (weather_data[\"lon\"] == lon)]\n",
    "\n",
    "        # Create the file name\n",
    "        dosya_adi = f\"koordinat_{lat}_{lon}.csv\"\n",
    "\n",
    "        # Write headers if the file does not exist, otherwise append data without headers\n",
    "        if not os.path.exists(dosya_adi):\n",
    "            coord_weather.to_csv(dosya_adi, index=False, header=True)  # Write headers for the first time\n",
    "        else:\n",
    "            coord_weather.to_csv(dosya_adi, mode='a', header=False, index=False)  # Do not write headers for subsequent appends\n",
    "\n",
    "        # Open the file and remove lat and lon columns\n",
    "        df = pd.read_csv(dosya_adi)\n",
    "        df = df.drop([\"lat\", \"lon\"], axis=1)\n",
    "        df.to_csv(dosya_adi, index=False)\n",
    "\n",
    "# Perform correlation analysis for each coordinate\n",
    "koordinat_korelasyonlari = {}\n",
    "for dosya_adi in glob.glob(\"koordinat_*.csv\"):\n",
    "    # Read the coordinate's data\n",
    "    koordinat_data = pd.read_csv(dosya_adi)\n",
    "    koordinat_data['date'] = pd.to_datetime(koordinat_data['date'])\n",
    "\n",
    "    # Filter data for common dates and hours\n",
    "    koordinat_data = koordinat_data[(koordinat_data[\"date\"].isin(common_dates)) & (koordinat_data[\"hour\"].isin(common_hours))]\n",
    "\n",
    "    # Merge production data with coordinate data\n",
    "    merged_data = pd.merge(\n",
    "        production_data[(production_data[\"date\"].isin(common_dates)) & (production_data[\"hour\"].isin(common_hours))],\n",
    "        koordinat_data,\n",
    "        on=[\"date\", \"hour\"]\n",
    "    )\n",
    "\n",
    "    # Calculate correlation for each weather variable\n",
    "    correlations = {}\n",
    "    for col in koordinat_data.columns:\n",
    "        if col in ['date', 'hour']:\n",
    "            continue\n",
    "        correlation = merged_data[\"production\"].corr(merged_data[col])\n",
    "        correlations[col] = correlation\n",
    "\n",
    "    # Save correlations for the coordinate\n",
    "    koordinat_korelasyonlari[dosya_adi] = correlations\n",
    "\n",
    "# Calculate influence factors for each variable for each coordinate\n",
    "koordinat_etkileri = {}\n",
    "for koordinat, korelasyonlar in koordinat_korelasyonlari.items():\n",
    "    koordinat_etkileri[koordinat] = {}\n",
    "    for degisken, korelasyon in korelasyonlar.items():\n",
    "        # Influence is calculated as the square of the absolute value of the correlation\n",
    "        etki = abs(korelasyon) ** 2\n",
    "        koordinat_etkileri[koordinat][degisken] = etki\n",
    "\n",
    "# Normalize influence factors for each variable\n",
    "for degisken in koordinat_etkileri[list(koordinat_etkileri.keys())[0]].keys():\n",
    "    etki_toplami = sum(koordinat_etkileri[koordinat][degisken] for koordinat in koordinat_etkileri)\n",
    "    for koordinat in koordinat_etkileri:\n",
    "        koordinat_etkileri[koordinat][degisken] /= etki_toplami\n",
    "\n",
    "# Convert influence factors to a DataFrame and print them\n",
    "etki_df = pd.DataFrame(koordinat_etkileri).T.reset_index()\n",
    "print(\"Koordinat Etki Çarpanları:\")\n",
    "print(etki_df.to_string(index=False))\n",
    "\n",
    "# Multiply each column in coordinate files by the corresponding influence factor and combine the results\n",
    "sonuc_df = pd.DataFrame()\n",
    "for dosya_adi in glob.glob(\"koordinat_*.csv\"):\n",
    "    # Read the coordinate data\n",
    "    koordinat_data = pd.read_csv(dosya_adi)\n",
    "    koordinat_data['date'] = pd.to_datetime(koordinat_data['date'])\n",
    "\n",
    "    # Get the influence factors for the coordinate\n",
    "    etkiler = koordinat_etkileri[dosya_adi]\n",
    "\n",
    "    # Multiply each column by the corresponding influence factor\n",
    "    for degisken, etki in etkiler.items():\n",
    "        koordinat_data[degisken] = koordinat_data[degisken] * etki\n",
    "\n",
    "    # Group by date and hour and sum the values\n",
    "    koordinat_data = koordinat_data.groupby(['date', 'hour']).sum().reset_index()\n",
    "\n",
    "    # Append to the result DataFrame\n",
    "    sonuc_df = pd.concat([sonuc_df, koordinat_data], ignore_index=True)\n",
    "\n",
    "# Group the results by date and hour and sum the values\n",
    "sonuc_df = sonuc_df.groupby(['date', 'hour']).sum().reset_index()\n",
    "\n",
    "# Round values to 3 decimal places where necessary\n",
    "for col in sonuc_df.columns:\n",
    "    if col in ['date', 'hour']:\n",
    "        continue\n",
    "    sonuc_df[col] = sonuc_df[col].round(3)\n",
    "\n",
    "# Save the final result to a CSV file\n",
    "sonuc_df.to_csv(\"agirlikli.csv\", index=False)\n",
    "\n",
    "# Remove the coordinate files as they are no longer needed\n",
    "for dosya_adi in glob.glob(\"koordinat_*.csv\"):\n",
    "    os.remove(dosya_adi)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Load the production data\n",
    "production_df = pd.read_csv(\"production.csv\")\n",
    "production_df['date'] = pd.to_datetime(production_df['date'])\n",
    "\n",
    "# Convert hour to string and combine with date to create a datetime column\n",
    "production_df[\"datetime\"] = pd.to_datetime(production_df['date'].dt.strftime('%Y-%m-%d') + ' ' + production_df[\"hour\"].astype(str) + ':00') \n",
    "\n",
    "# Load the weather data\n",
    "weather_df = pd.read_csv(\"processed_weather.csv\")\n",
    "weather_df['date'] = pd.to_datetime(weather_df['date'])\n",
    "weather_df[\"datetime\"] = pd.to_datetime(weather_df['date'].dt.strftime('%Y-%m-%d') + ' ' + weather_df[\"hour\"].astype(str) + ':00') \n",
    "\n",
    "# Load the weighted weather data\n",
    "weighted_df = pd.read_csv(\"agirlikli.csv\")\n",
    "weighted_df['date'] = pd.to_datetime(weighted_df['date'])\n",
    "weighted_df[\"datetime\"] = pd.to_datetime(weighted_df['date'].dt.strftime('%Y-%m-%d') + ' ' + weighted_df[\"hour\"].astype(str) + ':00') \n",
    "\n",
    "# Descriptive Analysis: Production Data\n",
    "\n",
    "# 1. Daily Production Averages\n",
    "daily_production_mean = production_df.groupby(production_df[\"datetime\"].dt.date)[\"production\"].mean()\n",
    "plt.figure(figsize=(12, 6))\n",
    "plt.plot(daily_production_mean.index, daily_production_mean.values, label=\"Average Daily Production\")\n",
    "plt.xlabel(\"Date\")\n",
    "plt.ylabel(\"Average Production (MWh)\")\n",
    "plt.title(\"Average Daily Solar Power Production\")\n",
    "plt.xticks(rotation=45) \n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# 2. Hourly Production Averages\n",
    "hourly_production_mean = production_df.groupby(production_df[\"datetime\"].dt.hour)[\"production\"].mean()\n",
    "plt.figure(figsize=(8, 4))\n",
    "plt.bar(hourly_production_mean.index, hourly_production_mean.values)\n",
    "plt.xlabel(\"Hour of Day\")\n",
    "plt.ylabel(\"Average Production (MWh)\")\n",
    "plt.title(\"Average Hourly Solar Power Production\")\n",
    "plt.show()\n",
    "\n",
    "# Descriptive Analysis: Weighted Weather Data\n",
    "weather_variables = ['dswrf_surface', 'tcdc_low.cloud.layer', 'tcdc_middle.cloud.layer',\n",
    "                   'tcdc_high.cloud.layer', 'tcdc_entire.atmosphere', 'uswrf_top_of_atmosphere',\n",
    "                   'csnow_surface', 'dlwrf_surface', 'uswrf_surface', 'tmp_surface']\n",
    "\n",
    "# Plot daily averages for each weather variable\n",
    "for variable in weather_variables:\n",
    "    # Calculate daily averages for the weighted variable\n",
    "    daily_weighted_mean = weighted_df.groupby(weighted_df[\"datetime\"].dt.date)[variable].mean()\n",
    "\n",
    "    # Plot the daily averages\n",
    "    plt.figure(figsize=(12, 6))\n",
    "    plt.plot(daily_weighted_mean.index, daily_weighted_mean.values, label=f\"Daily Average {variable.replace('.',' ')}\")\n",
    "    plt.xlabel(\"Date\")\n",
    "    plt.ylabel(f\"Daily Average {variable.replace('.',' ')}\")\n",
    "    plt.title(f\"Daily Average {variable.replace('.',' ')} Time Series\")\n",
    "    plt.xticks(rotation=45)\n",
    "    plt.legend()\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Load the weighted weather data\n",
    "weighted_df = pd.read_csv(\"agirlikli.csv\")\n",
    "weighted_df['date'] = pd.to_datetime(weighted_df['date'])\n",
    "weighted_df[\"datetime\"] = pd.to_datetime(weighted_df['date'].dt.strftime('%Y-%m-%d') + ' ' + weighted_df[\"hour\"].astype(str) + ':00')\n",
    "\n",
    "# Load the production data\n",
    "production_df = pd.read_csv(\"production.csv\")\n",
    "production_df['date'] = pd.to_datetime(production_df['date']) \n",
    "\n",
    "# Merge production data with weighted weather data on date and hour\n",
    "merged_df = pd.merge(production_df, weighted_df, on=[\"date\", \"hour\"], how=\"inner\")\n",
    "\n",
    "# Calculate the correlation of each weather variable with production\n",
    "correlations = merged_df.corr()[\"production\"].drop(\"production\")\n",
    "\n",
    "# Print the correlation values\n",
    "print(\"Correlations of Weighted Weather Data with Production:\")\n",
    "print(correlations)\n",
    "\n",
    "# Plot the correlation heatmap\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.heatmap(correlations.to_frame(), annot=True, cmap=\"coolwarm\")\n",
    "plt.title(\"Correlation Heatmap: Production vs. Weighted Weather Data\")\n",
    "plt.show()\n",
    "\n",
    "# List of weather variables to analyze\n",
    "weather_variables = ['dswrf_surface', 'tcdc_low.cloud.layer', 'tcdc_middle.cloud.layer',\n",
    "                   'tcdc_high.cloud.layer', 'tcdc_entire.atmosphere', 'uswrf_top_of_atmosphere',\n",
    "                   'csnow_surface', 'dlwrf_surface', 'uswrf_surface', 'tmp_surface']\n",
    "\n",
    "# Analyze the correlation between daily averages of weather variables and production\n",
    "for variable in weather_variables:\n",
    "    # Merge production and weighted weather data\n",
    "    merged_df = pd.merge(production_df, weighted_df, on=[\"date\", \"hour\"], how=\"inner\")\n",
    "\n",
    "    # Filter data to only include common dates\n",
    "    common_dates = set(production_df[\"date\"].unique()) & set(weighted_df[\"date\"].unique())\n",
    "    merged_df = merged_df[merged_df[\"date\"].isin(common_dates)]\n",
    "\n",
    "    # Calculate daily averages for production and the weather variable\n",
    "    daily_production_mean = merged_df.groupby(merged_df[\"date\"])[\"production\"].mean()\n",
    "    daily_weighted_mean = merged_df.groupby(merged_df[\"date\"])[variable].mean()\n",
    "\n",
    "    # Create a scatter plot of daily averages\n",
    "    plt.figure(figsize=(10, 6))\n",
    "    plt.scatter(daily_weighted_mean, daily_production_mean)\n",
    "    plt.xlabel(f\"Daily Average {variable.replace('.',' ')}\")\n",
    "    plt.ylabel(\"Daily Average Production (MWh)\")\n",
    "    plt.title(f\"Daily Average Production vs. Daily Average {variable.replace('.',' ')}\")\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from statsmodels.graphics.tsaplots import plot_acf, plot_pacf\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Load the production data\n",
    "production_df = pd.read_csv(\"production.csv\")\n",
    "\n",
    "# Load the weighted weather data\n",
    "averages_df = pd.read_csv(\"agirlikli.csv\")\n",
    "\n",
    "\n",
    "# Ensure 'datetime' column exists in both dataframes\n",
    "if 'date' in production_df.columns and 'hour' in production_df.columns:\n",
    "    production_df[\"datetime\"] = pd.to_datetime(production_df[\"date\"] + \" \" + production_df[\"hour\"].astype(str) + \":00\")\n",
    "\n",
    "if 'date' in averages_df.columns and 'hour' in averages_df.columns:\n",
    "    averages_df[\"datetime\"] = pd.to_datetime(averages_df[\"date\"] + \" \" + averages_df[\"hour\"].astype(str) + \":00\")\n",
    "\n",
    "\n",
    "\n",
    "# Merge production data with weighted weather data on datetime\n",
    "merged_df = pd.merge(production_df, averages_df, on=\"datetime\", how=\"inner\")\n",
    "\n",
    "# Plot the Autocorrelation Function (ACF) for the production data\n",
    "plot_acf(merged_df[\"production\"], lags=50)  \n",
    "plt.show()\n",
    "\n",
    "# Plot the Partial Autocorrelation Function (PACF) for the production data\n",
    "plot_pacf(merged_df[\"production\"], lags=50)  \n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pmdarima import auto_arima\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_percentage_error\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "# Load data\n",
    "production_df = pd.read_csv(\"production.csv\")\n",
    "weather_df = pd.read_csv(\"agirlikli.csv\")\n",
    "\n",
    "# Convert date columns to datetime\n",
    "production_df[\"datetime\"] = pd.to_datetime(production_df[\"date\"] + \" \" + production_df[\"hour\"].astype(str) + \":00\")\n",
    "weather_df[\"datetime\"] = pd.to_datetime(weather_df[\"date\"] + \" \" + weather_df[\"hour\"].astype(str) + \":00\")\n",
    "\n",
    "# Merge data on datetime\n",
    "merged_df = pd.merge(production_df, weather_df, on=\"datetime\", how=\"inner\")\n",
    "\n",
    "# Drop unnecessary columns\n",
    "merged_df = merged_df.drop(columns=[\"date_x\", \"hour_x\", \"date_y\", \"hour_y\"])\n",
    "\n",
    "# Fill missing values\n",
    "merged_df = merged_df.ffill()\n",
    "\n",
    "# Define features and target\n",
    "exog_features = ['dswrf_surface', 'tcdc_low.cloud.layer', 'tcdc_middle.cloud.layer',\n",
    "                 'tcdc_high.cloud.layer', 'tcdc_entire.atmosphere', 'uswrf_top_of_atmosphere',\n",
    "                 'csnow_surface', 'dlwrf_surface', 'uswrf_surface', 'tmp_surface']\n",
    "target = 'production'\n",
    "\n",
    "# Function to train ARIMA model for a specific hour\n",
    "def train_arima_for_hour(hour):\n",
    "    # Filter data for the specific hour\n",
    "    df_hour = merged_df[merged_df['datetime'].dt.hour == hour].copy()\n",
    "\n",
    "    # Return None if no data for the hour\n",
    "    if df_hour.empty:\n",
    "        return hour, None\n",
    "\n",
    "    # Train ARIMA model with exogenous features\n",
    "    model = auto_arima(df_hour[target], exogenous=df_hour[exog_features], seasonal=True, m=24, \n",
    "                       stepwise=True, suppress_warnings=True, error_action='ignore',\n",
    "                       max_p=2, max_q=2, max_P=1, max_Q=1, max_order=4, max_d=1, max_D=1)\n",
    "\n",
    "    return hour, model\n",
    "\n",
    "# Train ARIMA models in parallel for each hour\n",
    "models = dict(Parallel(n_jobs=-1)(delayed(train_arima_for_hour)(hour) for hour in range(24)))\n",
    "\n",
    "# Forecasting for May 25, 2024\n",
    "forecast_date = pd.date_range('2024-05-25', periods=24, freq='H')\n",
    "forecast_df = pd.DataFrame({'datetime': forecast_date})\n",
    "\n",
    "# Add exogenous features to the forecast dataframe\n",
    "for feature in exog_features:\n",
    "    forecast_df[feature] = weather_df[weather_df['datetime'].isin(forecast_date)][feature].values\n",
    "\n",
    "# List to store predictions\n",
    "all_predictions = []\n",
    "for hour in range(24):\n",
    "    model = models.get(hour)\n",
    "    if model is None:\n",
    "        all_predictions.append([0])\n",
    "        continue\n",
    "\n",
    "    # Forecast for the specific hour\n",
    "    hourly_forecast = forecast_df[forecast_df['datetime'].dt.hour == hour]\n",
    "    forecast = model.predict(n_periods=1, exogenous=hourly_forecast[exog_features])\n",
    "    forecast = np.clip(forecast, a_min=0, a_max=None)\n",
    "    all_predictions.append(forecast)\n",
    "\n",
    "# Flatten predictions and calculate error metrics\n",
    "flattened_predictions = [pred for sublist in all_predictions for pred in sublist]\n",
    "\n",
    "# Ensure the prediction length matches the actuals\n",
    "actuals = merged_df[merged_df['datetime'].dt.date == pd.Timestamp('2024-05-23').date()]['production'].values[:len(flattened_predictions)]\n",
    "mse = mean_squared_error(actuals, flattened_predictions)\n",
    "rmse = mse ** 0.5\n",
    "mape = mean_absolute_percentage_error(actuals, flattened_predictions)\n",
    "\n",
    "# Print the forecast for May 25, 2024\n",
    "print(\"\\n2024-05-25 Production Forecast:\")\n",
    "for hour, predictions in enumerate(all_predictions):\n",
    "    for prediction in predictions:\n",
    "        print(f\"  Hour {hour:02d}: {prediction:.2f}\")\n",
    "\n",
    "# Print error metrics\n",
    "print(f\"MSE: {mse:.2f}\")\n",
    "print(f\"RMSE: {rmse:.2f}\")\n",
    "print(f\"MAPE: {mape:.2f}\")\n",
    "\n",
    "# Optional: Print hourly predictions in a single line\n",
    "hourly_predictions_str = \", \".join([f\"{prediction:.2f}\" if prediction != 0 else \"0\" for predictions in all_predictions for prediction in predictions])\n",
    "print(hourly_predictions_str)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}