ANN_SBO(mealpy)_airfoil_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4cd1c3db",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import modules\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler, QuantileTransformer, MaxAbsScaler\n",
    "from sklearn.metrics import r2_score, mean_absolute_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7b7066bd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: mealpy in c:\\users\\nithi\\anaconda3\\lib\\site-packages (3.0.1)\n",
      "Requirement already satisfied: numpy>=1.17.5 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from mealpy) (1.24.3)\n",
      "Requirement already satisfied: matplotlib>=3.3.0 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from mealpy) (3.7.1)\n",
      "Requirement already satisfied: scipy>=1.7.1 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from mealpy) (1.10.1)\n",
      "Requirement already satisfied: pandas>=1.2.0 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from mealpy) (1.5.3)\n",
      "Requirement already satisfied: opfunu>=1.0.0 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from mealpy) (1.0.1)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from matplotlib>=3.3.0->mealpy) (1.0.5)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from matplotlib>=3.3.0->mealpy) (0.11.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from matplotlib>=3.3.0->mealpy) (4.25.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from matplotlib>=3.3.0->mealpy) (1.4.4)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from matplotlib>=3.3.0->mealpy) (23.0)\n",
      "Requirement already satisfied: pillow>=6.2.0 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from matplotlib>=3.3.0->mealpy) (9.4.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from matplotlib>=3.3.0->mealpy) (3.0.9)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from matplotlib>=3.3.0->mealpy) (2.8.2)\n",
      "Requirement already satisfied: requests>=2.27.0 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from opfunu>=1.0.0->mealpy) (2.31.0)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from pandas>=1.2.0->mealpy) (2022.7)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from python-dateutil>=2.7->matplotlib>=3.3.0->mealpy) (1.16.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from requests>=2.27.0->opfunu>=1.0.0->mealpy) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from requests>=2.27.0->opfunu>=1.0.0->mealpy) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from requests>=2.27.0->opfunu>=1.0.0->mealpy) (1.26.16)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\nithi\\anaconda3\\lib\\site-packages (from requests>=2.27.0->opfunu>=1.0.0->mealpy) (2023.7.22)\n"
     ]
    }
   ],
   "source": [
    "!pip install mealpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "71ceeff6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Input, Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "114db60b",
   "metadata": {},
   "outputs": [],
   "source": [
    "url_file = 'https://archive.ics.uci.edu/ml/machine-learning-databases/00291/airfoil_self_noise.dat'\n",
    "columns = ['frequency',\n",
    "           'angle_of_attack', \n",
    "           'chord_length', \n",
    "           'free_stream_velocity',\n",
    "           'suction_side_displacement_thickness',\n",
    "           'scaled_sound_pressure_level']\n",
    "\n",
    "features = ['frequency','angle_of_attack', \n",
    "            'chord_length', \n",
    "            'free_stream_velocity',\n",
    "            'suction_side_displacement_thickness']\n",
    "\n",
    "df = pd.read_csv(url_file, sep='\\t', header=None, names=columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0c6abd23",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>frequency</th>\n",
       "      <th>angle_of_attack</th>\n",
       "      <th>chord_length</th>\n",
       "      <th>free_stream_velocity</th>\n",
       "      <th>suction_side_displacement_thickness</th>\n",
       "      <th>scaled_sound_pressure_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>800</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>126.201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>125.201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1250</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>125.951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1600</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>127.591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.3048</td>\n",
       "      <td>71.3</td>\n",
       "      <td>0.002663</td>\n",
       "      <td>127.461</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   frequency  angle_of_attack  chord_length  free_stream_velocity  \\\n",
       "0        800              0.0        0.3048                  71.3   \n",
       "1       1000              0.0        0.3048                  71.3   \n",
       "2       1250              0.0        0.3048                  71.3   \n",
       "3       1600              0.0        0.3048                  71.3   \n",
       "4       2000              0.0        0.3048                  71.3   \n",
       "\n",
       "   suction_side_displacement_thickness  scaled_sound_pressure_level  \n",
       "0                             0.002663                      126.201  \n",
       "1                             0.002663                      125.201  \n",
       "2                             0.002663                      125.951  \n",
       "3                             0.002663                      127.591  \n",
       "4                             0.002663                      127.461  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b927a7dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>frequency</th>\n",
       "      <th>angle_of_attack</th>\n",
       "      <th>chord_length</th>\n",
       "      <th>free_stream_velocity</th>\n",
       "      <th>suction_side_displacement_thickness</th>\n",
       "      <th>scaled_sound_pressure_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1498</th>\n",
       "      <td>2500</td>\n",
       "      <td>15.6</td>\n",
       "      <td>0.1016</td>\n",
       "      <td>39.6</td>\n",
       "      <td>0.052849</td>\n",
       "      <td>110.264</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1499</th>\n",
       "      <td>3150</td>\n",
       "      <td>15.6</td>\n",
       "      <td>0.1016</td>\n",
       "      <td>39.6</td>\n",
       "      <td>0.052849</td>\n",
       "      <td>109.254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1500</th>\n",
       "      <td>4000</td>\n",
       "      <td>15.6</td>\n",
       "      <td>0.1016</td>\n",
       "      <td>39.6</td>\n",
       "      <td>0.052849</td>\n",
       "      <td>106.604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1501</th>\n",
       "      <td>5000</td>\n",
       "      <td>15.6</td>\n",
       "      <td>0.1016</td>\n",
       "      <td>39.6</td>\n",
       "      <td>0.052849</td>\n",
       "      <td>106.224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1502</th>\n",
       "      <td>6300</td>\n",
       "      <td>15.6</td>\n",
       "      <td>0.1016</td>\n",
       "      <td>39.6</td>\n",
       "      <td>0.052849</td>\n",
       "      <td>104.204</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      frequency  angle_of_attack  chord_length  free_stream_velocity  \\\n",
       "1498       2500             15.6        0.1016                  39.6   \n",
       "1499       3150             15.6        0.1016                  39.6   \n",
       "1500       4000             15.6        0.1016                  39.6   \n",
       "1501       5000             15.6        0.1016                  39.6   \n",
       "1502       6300             15.6        0.1016                  39.6   \n",
       "\n",
       "      suction_side_displacement_thickness  scaled_sound_pressure_level  \n",
       "1498                             0.052849                      110.264  \n",
       "1499                             0.052849                      109.254  \n",
       "1500                             0.052849                      106.604  \n",
       "1501                             0.052849                      106.224  \n",
       "1502                             0.052849                      104.204  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ba40ec39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>frequency</th>\n",
       "      <th>angle_of_attack</th>\n",
       "      <th>chord_length</th>\n",
       "      <th>free_stream_velocity</th>\n",
       "      <th>suction_side_displacement_thickness</th>\n",
       "      <th>scaled_sound_pressure_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1503.000000</td>\n",
       "      <td>1503.000000</td>\n",
       "      <td>1503.000000</td>\n",
       "      <td>1503.000000</td>\n",
       "      <td>1503.000000</td>\n",
       "      <td>1503.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>2886.380572</td>\n",
       "      <td>6.782302</td>\n",
       "      <td>0.136548</td>\n",
       "      <td>50.860745</td>\n",
       "      <td>0.011140</td>\n",
       "      <td>124.835943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3152.573137</td>\n",
       "      <td>5.918128</td>\n",
       "      <td>0.093541</td>\n",
       "      <td>15.572784</td>\n",
       "      <td>0.013150</td>\n",
       "      <td>6.898657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>200.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.025400</td>\n",
       "      <td>31.700000</td>\n",
       "      <td>0.000401</td>\n",
       "      <td>103.380000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>800.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.050800</td>\n",
       "      <td>39.600000</td>\n",
       "      <td>0.002535</td>\n",
       "      <td>120.191000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1600.000000</td>\n",
       "      <td>5.400000</td>\n",
       "      <td>0.101600</td>\n",
       "      <td>39.600000</td>\n",
       "      <td>0.004957</td>\n",
       "      <td>125.721000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>4000.000000</td>\n",
       "      <td>9.900000</td>\n",
       "      <td>0.228600</td>\n",
       "      <td>71.300000</td>\n",
       "      <td>0.015576</td>\n",
       "      <td>129.995500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>20000.000000</td>\n",
       "      <td>22.200000</td>\n",
       "      <td>0.304800</td>\n",
       "      <td>71.300000</td>\n",
       "      <td>0.058411</td>\n",
       "      <td>140.987000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          frequency  angle_of_attack  chord_length  free_stream_velocity  \\\n",
       "count   1503.000000      1503.000000   1503.000000           1503.000000   \n",
       "mean    2886.380572         6.782302      0.136548             50.860745   \n",
       "std     3152.573137         5.918128      0.093541             15.572784   \n",
       "min      200.000000         0.000000      0.025400             31.700000   \n",
       "25%      800.000000         2.000000      0.050800             39.600000   \n",
       "50%     1600.000000         5.400000      0.101600             39.600000   \n",
       "75%     4000.000000         9.900000      0.228600             71.300000   \n",
       "max    20000.000000        22.200000      0.304800             71.300000   \n",
       "\n",
       "       suction_side_displacement_thickness  scaled_sound_pressure_level  \n",
       "count                          1503.000000                  1503.000000  \n",
       "mean                              0.011140                   124.835943  \n",
       "std                               0.013150                     6.898657  \n",
       "min                               0.000401                   103.380000  \n",
       "25%                               0.002535                   120.191000  \n",
       "50%                               0.004957                   125.721000  \n",
       "75%                               0.015576                   129.995500  \n",
       "max                               0.058411                   140.987000  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cfc7d504",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1503 entries, 0 to 1502\n",
      "Data columns (total 6 columns):\n",
      " #   Column                               Non-Null Count  Dtype  \n",
      "---  ------                               --------------  -----  \n",
      " 0   frequency                            1503 non-null   int64  \n",
      " 1   angle_of_attack                      1503 non-null   float64\n",
      " 2   chord_length                         1503 non-null   float64\n",
      " 3   free_stream_velocity                 1503 non-null   float64\n",
      " 4   suction_side_displacement_thickness  1503 non-null   float64\n",
      " 5   scaled_sound_pressure_level          1503 non-null   float64\n",
      "dtypes: float64(5), int64(1)\n",
      "memory usage: 70.6 KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d5f5320f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_input(df):\n",
    "    \n",
    "    train_data = df.copy()\n",
    "    train_set = train_data.drop(\"scaled_sound_pressure_level\", axis = 1)\n",
    "    train_labels = train_data[\"scaled_sound_pressure_level\"].copy()\n",
    "    \n",
    "    train_set = QuantileTransformer(random_state=0).fit_transform(train_set)\n",
    "    train_set = MaxAbsScaler().fit_transform(train_set)\n",
    "    train_set = StandardScaler().fit_transform(train_set)\n",
    "    \n",
    "    # Train-test split\n",
    "    X_train, X_test, y_train, y_test = train_test_split(train_set, train_labels, test_size=0.2)\n",
    "    \n",
    "    return X_train, X_test, y_train, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "b577ebf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = preprocess_input(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d486047c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1202, 5)\n",
      "(301, 5)\n",
      "(1202,)\n",
      "(301,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5b3219e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " dense (Dense)               (None, 32)                192       \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 1)                 33        \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 225 (900.00 Byte)\n",
      "Trainable params: 225 (900.00 Byte)\n",
      "Non-trainable params: 0 (0.00 Byte)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(Input(X_train.shape[1],))\n",
    "\n",
    "model.add(Dense(32, activation = 'relu'))\n",
    "\n",
    "model.add(Dense(1))\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5f0f3766",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss = 'mean_squared_error', metrics = tf.keras.metrics.RootMeanSquaredError())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "38595e5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_shape(model):\n",
    "    weights_layer = model.get_weights()\n",
    "    shapes = []\n",
    "    for weights in weights_layer:\n",
    "        shapes.append(weights.shape)\n",
    "    return shapes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7c976809",
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_shape(weights,shapes):\n",
    "    new_weights = []\n",
    "    index=0\n",
    "    for shape in shapes:\n",
    "        if(len(shape)>1):\n",
    "            n_nodes = np.prod(shape) + index\n",
    "        else:\n",
    "            n_nodes=shape[0]+index\n",
    "        tmp = np.array(weights[index:n_nodes]).reshape(shape)\n",
    "        new_weights.append(tmp)\n",
    "        index=n_nodes\n",
    "    return new_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "de82e63a",
   "metadata": {},
   "outputs": [],
   "source": [
    "shape = get_shape(model)\n",
    "num_solutions = 15\n",
    "dimensions = 225"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0d660063",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_nn(W):\n",
    "    model.set_weights(set_shape(W,shape))\n",
    "    score = model.evaluate(X_train, y_train, verbose = 0)\n",
    "    return score[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e51d85f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from mealpy import SBO, FloatVar\n",
    "lb = [-10 for i in range(dimensions)]\n",
    "ub = [10 for i in range(dimensions)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4ff95e38",
   "metadata": {},
   "outputs": [],
   "source": [
    "problem_dict = {\n",
    "    \"bounds\": FloatVar(lb = lb, ub = ub, name=\"NeuralNetwork\"),\n",
    "    \"minmax\": \"min\",\n",
    "    \"obj_func\": evaluate_nn\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d13a5e69",
   "metadata": {},
   "outputs": [],
   "source": [
    "SBO_optimizer = SBO.DevSBO(epoch=1000, pop_size=15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3f747770",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/02/28 01:26:37 PM, INFO, mealpy.bio_based.SBO.DevSBO: Solving single objective optimization problem.\n",
      "2024/02/28 01:26:42 PM, INFO, mealpy.bio_based.SBO.DevSBO: >>>Problem: P, Epoch: 1, Current best: 102.54239654541016, Global best: 102.54239654541016, Runtime: 2.74168 seconds\n",
      "2024/02/28 01:26:45 PM, INFO, mealpy.bio_based.SBO.DevSBO: >>>Problem: P, Epoch: 2, Current best: 74.64832305908203, Global best: 74.64832305908203, Runtime: 3.12388 seconds\n",
      "2024/02/28 01:26:48 PM, INFO, mealpy.bio_based.SBO.DevSBO: >>>Problem: P, Epoch: 3, Current best: 68.74332427978516, Global best: 68.74332427978516, Runtime: 2.98745 seconds\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[24], line 1\u001b[0m\n\u001b[1;32m----> 1\u001b[0m g_best \u001b[38;5;241m=\u001b[39m SBO_optimizer\u001b[38;5;241m.\u001b[39msolve(problem_dict)\n",
      "File \u001b[1;32m~\\anaconda3\\Lib\\site-packages\\mealpy\\optimizer.py:237\u001b[0m, in \u001b[0;36mOptimizer.solve\u001b[1;34m(self, problem, mode, n_workers, termination, starting_solutions, seed)\u001b[0m\n\u001b[0;32m    234\u001b[0m time_epoch \u001b[38;5;241m=\u001b[39m time\u001b[38;5;241m.\u001b[39mperf_counter()\n\u001b[0;32m    236\u001b[0m \u001b[38;5;66;03m## Evolve method will be called in child class\u001b[39;00m\n\u001b[1;32m--> 237\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mevolve(epoch)\n\u001b[0;32m    239\u001b[0m \u001b[38;5;66;03m# Update global best solution, the population is sorted or not depended on algorithm's strategy\u001b[39;00m\n\u001b[0;32m    240\u001b[0m pop_temp, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mg_best \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mupdate_global_best_agent(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpop)\n",
      "File \u001b[1;32m~\\anaconda3\\Lib\\site-packages\\mealpy\\bio_based\\SBO.py:92\u001b[0m, in \u001b[0;36mDevSBO.evolve\u001b[1;34m(self, epoch)\u001b[0m\n\u001b[0;32m     90\u001b[0m     pop_new\u001b[38;5;241m.\u001b[39mappend(agent)\n\u001b[0;32m     91\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmode \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mAVAILABLE_MODES:\n\u001b[1;32m---> 92\u001b[0m         agent\u001b[38;5;241m.\u001b[39mtarget \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_target(pos_new)\n\u001b[0;32m     93\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpop[idx] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_better_agent(agent, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpop[idx], \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mproblem\u001b[38;5;241m.\u001b[39mminmax)\n\u001b[0;32m     94\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmode \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mAVAILABLE_MODES:\n",
      "File \u001b[1;32m~\\anaconda3\\Lib\\site-packages\\mealpy\\optimizer.py:406\u001b[0m, in \u001b[0;36mOptimizer.get_target\u001b[1;34m(self, solution, counted)\u001b[0m\n\u001b[0;32m    404\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m counted:\n\u001b[0;32m    405\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnfe_counter \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m--> 406\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mproblem\u001b[38;5;241m.\u001b[39mget_target(solution)\n",
      "File \u001b[1;32m~\\anaconda3\\Lib\\site-packages\\mealpy\\utils\\problem.py:200\u001b[0m, in \u001b[0;36mProblem.get_target\u001b[1;34m(self, solution)\u001b[0m\n\u001b[0;32m    192\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_target\u001b[39m(\u001b[38;5;28mself\u001b[39m, solution: np\u001b[38;5;241m.\u001b[39mndarray) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Target:\n\u001b[0;32m    193\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    194\u001b[0m \u001b[38;5;124;03m    Args:\u001b[39;00m\n\u001b[0;32m    195\u001b[0m \u001b[38;5;124;03m        solution: The real-value solution\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    198\u001b[0m \u001b[38;5;124;03m        The target object\u001b[39;00m\n\u001b[0;32m    199\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 200\u001b[0m     objs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobj_func(solution)\n\u001b[0;32m    201\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m Target(objectives\u001b[38;5;241m=\u001b[39mobjs, weights\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobj_weights)\n",
      "Cell \u001b[1;32mIn[17], line 3\u001b[0m, in \u001b[0;36mevaluate_nn\u001b[1;34m(W)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mevaluate_nn\u001b[39m(W):\n\u001b[0;32m      2\u001b[0m     model\u001b[38;5;241m.\u001b[39mset_weights(set_shape(W,shape))\n\u001b[1;32m----> 3\u001b[0m     score \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mevaluate(X_train, y_train, verbose \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m)\n\u001b[0;32m      4\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m score[\u001b[38;5;241m1\u001b[39m]\n",
      "File \u001b[1;32m~\\anaconda3\\Lib\\site-packages\\keras\\src\\utils\\traceback_utils.py:65\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     63\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m     64\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m---> 65\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m fn(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m     66\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[1;32m~\\anaconda3\\Lib\\site-packages\\keras\\src\\engine\\training.py:2261\u001b[0m, in \u001b[0;36mModel.evaluate\u001b[1;34m(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, return_dict, **kwargs)\u001b[0m\n\u001b[0;32m   2259\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39menable_tune_steps_per_execution:\n\u001b[0;32m   2260\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_steps_per_execution_tuner\u001b[38;5;241m.\u001b[39mstart()\n\u001b[1;32m-> 2261\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m (\n\u001b[0;32m   2262\u001b[0m     _,\n\u001b[0;32m   2263\u001b[0m     dataset_or_iterator,\n\u001b[0;32m   2264\u001b[0m ) \u001b[38;5;129;01min\u001b[39;00m data_handler\u001b[38;5;241m.\u001b[39menumerate_epochs():  \u001b[38;5;66;03m# Single epoch.\u001b[39;00m\n\u001b[0;32m   2265\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreset_metrics()\n\u001b[0;32m   2266\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m data_handler\u001b[38;5;241m.\u001b[39mcatch_stop_iteration():\n",
      "File \u001b[1;32m~\\anaconda3\\Lib\\site-packages\\keras\\src\\engine\\data_adapter.py:1341\u001b[0m, in \u001b[0;36mDataHandler.enumerate_epochs\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1339\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Yields `(epoch, tf.data.Iterator)`.\"\"\"\u001b[39;00m\n\u001b[0;32m   1340\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_truncate_execution_to_epoch():\n\u001b[1;32m-> 1341\u001b[0m     data_iterator \u001b[38;5;241m=\u001b[39m \u001b[38;5;28miter\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_dataset)\n\u001b[0;32m   1342\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m epoch \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_initial_epoch, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_epochs):\n\u001b[0;32m   1343\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_insufficient_data:  \u001b[38;5;66;03m# Set by `catch_stop_iteration`.\u001b[39;00m\n",
      "File \u001b[1;32m~\\anaconda3\\Lib\\site-packages\\tensorflow\\python\\data\\ops\\dataset_ops.py:496\u001b[0m, in \u001b[0;36mDatasetV2.__iter__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    494\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m context\u001b[38;5;241m.\u001b[39mexecuting_eagerly() \u001b[38;5;129;01mor\u001b[39;00m ops\u001b[38;5;241m.\u001b[39minside_function():\n\u001b[0;32m    495\u001b[0m   \u001b[38;5;28;01mwith\u001b[39;00m ops\u001b[38;5;241m.\u001b[39mcolocate_with(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_variant_tensor):\n\u001b[1;32m--> 496\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m iterator_ops\u001b[38;5;241m.\u001b[39mOwnedIterator(\u001b[38;5;28mself\u001b[39m)\n\u001b[0;32m    497\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m    498\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`tf.data.Dataset` only supports Python-style \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    499\u001b[0m                      \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124miteration in eager mode or within tf.function.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "File \u001b[1;32m~\\anaconda3\\Lib\\site-packages\\tensorflow\\python\\data\\ops\\iterator_ops.py:705\u001b[0m, in \u001b[0;36mOwnedIterator.__init__\u001b[1;34m(self, dataset, components, element_spec)\u001b[0m\n\u001b[0;32m    701\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m (components \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mor\u001b[39;00m element_spec \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m    702\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m    703\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mWhen `dataset` is provided, `element_spec` and `components` must \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    704\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnot be specified.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m--> 705\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_create_iterator(dataset)\n\u001b[0;32m    707\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_next_call_count \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n",
      "File \u001b[1;32m~\\anaconda3\\Lib\\site-packages\\tensorflow\\python\\data\\ops\\iterator_ops.py:744\u001b[0m, in \u001b[0;36mOwnedIterator._create_iterator\u001b[1;34m(self, dataset)\u001b[0m\n\u001b[0;32m    741\u001b[0m   \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(fulltype\u001b[38;5;241m.\u001b[39margs[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39margs[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39margs) \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mlen\u001b[39m(\n\u001b[0;32m    742\u001b[0m       \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_flat_output_types)\n\u001b[0;32m    743\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_iterator_resource\u001b[38;5;241m.\u001b[39mop\u001b[38;5;241m.\u001b[39mexperimental_set_type(fulltype)\n\u001b[1;32m--> 744\u001b[0m gen_dataset_ops\u001b[38;5;241m.\u001b[39mmake_iterator(ds_variant, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_iterator_resource)\n",
      "File \u001b[1;32m~\\anaconda3\\Lib\\site-packages\\tensorflow\\python\\ops\\gen_dataset_ops.py:3451\u001b[0m, in \u001b[0;36mmake_iterator\u001b[1;34m(dataset, iterator, name)\u001b[0m\n\u001b[0;32m   3449\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m tld\u001b[38;5;241m.\u001b[39mis_eager:\n\u001b[0;32m   3450\u001b[0m   \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m-> 3451\u001b[0m     _result \u001b[38;5;241m=\u001b[39m pywrap_tfe\u001b[38;5;241m.\u001b[39mTFE_Py_FastPathExecute(\n\u001b[0;32m   3452\u001b[0m       _ctx, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mMakeIterator\u001b[39m\u001b[38;5;124m\"\u001b[39m, name, dataset, iterator)\n\u001b[0;32m   3453\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m _result\n\u001b[0;32m   3454\u001b[0m   \u001b[38;5;28;01mexcept\u001b[39;00m _core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "g_best = SBO_optimizer.solve(problem_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "74579590",
   "metadata": {},
   "outputs": [],
   "source": [
    "solution = g_best.solution\n",
    "model.set_weights(set_shape(solution, shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c80edfc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# RMSE\n",
    "score = model.evaluate(X_train, y_train)\n",
    "train_rmse, train_mse = score[1], score[0]\n",
    "\n",
    "score = model.evaluate(X_test, y_test)\n",
    "test_rmse, test_mse = score[1], score[0]\n",
    "\n",
    "print(\"Train RMSE = \", train_rmse)\n",
    "print(\"Test RMSE = \", test_rmse)\n",
    "print(\"---------------------------\")\n",
    "\n",
    "#MSE\n",
    "print(\"Train MSE = \", train_mse)\n",
    "print(\"Test MSE = \", test_mse)\n",
    "print(\"---------------------------\")\n",
    "\n",
    "#R2 Score\n",
    "y_train_pred = model.predict(X_train)\n",
    "y_test_pred = model.predict(X_test)\n",
    "train_r2_score = r2_score(y_train, y_train_pred)\n",
    "test_r2_score = r2_score(y_test, y_test_pred)\n",
    "\n",
    "print(\"Train R2 = \", train_r2_score)\n",
    "print(\"Test R2 = \", test_r2_score)\n",
    "print(\"-----------------------------\")\n",
    "\n",
    "# MAE\n",
    "train_mae = mean_absolute_error(y_train, y_train_pred)\n",
    "test_mae = mean_absolute_error(y_test, y_test_pred)\n",
    "print(\"Train MAE = \", train_mae)\n",
    "print(\"Test MAE = \", test_mae)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a04d477",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}