modelo4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "249dcaee",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:21.154860Z",
     "start_time": "2023-04-20T00:45:19.862508Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import datetime\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from scipy.sparse import hstack, vstack\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics import roc_auc_score, average_precision_score\n",
    "from lightgbm import LGBMClassifier\n",
    "from skopt import forest_minimize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5c5fba7c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:22.061610Z",
     "start_time": "2023-04-20T00:45:22.020243Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1314, 14)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('dados/silver/dados_tratados_com_features_novas_com_todas_labels.csv', index_col=0).dropna(subset=['label'])\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "921300d0",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:22.739429Z",
     "start_time": "2023-04-20T00:45:22.707336Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>titulo</th>\n",
       "      <th>label</th>\n",
       "      <th>canal</th>\n",
       "      <th>data_upload</th>\n",
       "      <th>categoria</th>\n",
       "      <th>tags</th>\n",
       "      <th>duracao_segundos</th>\n",
       "      <th>link</th>\n",
       "      <th>visualizacoes</th>\n",
       "      <th>quantidade_likes</th>\n",
       "      <th>query</th>\n",
       "      <th>tempo_desde_pub</th>\n",
       "      <th>visualizacoes_por_dia</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>watch?v=kE9gYQDyVr0</td>\n",
       "      <td>Trabalhando com Bases de Dados (Datasets) Desb...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Hashtag Programação</td>\n",
       "      <td>2023-04-17</td>\n",
       "      <td>['Education']</td>\n",
       "      <td>['Bases de dados desbalanceadas', 'Datasets de...</td>\n",
       "      <td>1326.0</td>\n",
       "      <td>https://www.youtube.com/watch?v=kE9gYQDyVr0</td>\n",
       "      <td>858</td>\n",
       "      <td>72.0</td>\n",
       "      <td>ciencia+de+dados</td>\n",
       "      <td>3.0</td>\n",
       "      <td>286.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>watch?v=GgEUPkAG1ho</td>\n",
       "      <td>Trabalho ciencia de dados-parte-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Jonas Silva</td>\n",
       "      <td>2023-04-13</td>\n",
       "      <td>['People &amp; Blogs']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>389.0</td>\n",
       "      <td>https://www.youtube.com/watch?v=GgEUPkAG1ho</td>\n",
       "      <td>2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>ciencia+de+dados</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.285714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    id                                             titulo  \\\n",
       "0  watch?v=kE9gYQDyVr0  Trabalhando com Bases de Dados (Datasets) Desb...   \n",
       "1  watch?v=GgEUPkAG1ho                  Trabalho ciencia de dados-parte-1   \n",
       "\n",
       "   label                canal data_upload           categoria  \\\n",
       "0    0.0  Hashtag Programação  2023-04-17       ['Education']   \n",
       "1    0.0          Jonas Silva  2023-04-13  ['People & Blogs']   \n",
       "\n",
       "                                                tags  duracao_segundos  \\\n",
       "0  ['Bases de dados desbalanceadas', 'Datasets de...            1326.0   \n",
       "1                                                NaN             389.0   \n",
       "\n",
       "                                          link  visualizacoes  \\\n",
       "0  https://www.youtube.com/watch?v=kE9gYQDyVr0            858   \n",
       "1  https://www.youtube.com/watch?v=GgEUPkAG1ho              2   \n",
       "\n",
       "   quantidade_likes             query  tempo_desde_pub  visualizacoes_por_dia  \n",
       "0              72.0  ciencia+de+dados              3.0             286.000000  \n",
       "1               0.0  ciencia+de+dados              7.0               0.285714  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5406749b",
   "metadata": {},
   "source": [
    "# Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2688023e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:24.090268Z",
     "start_time": "2023-04-20T00:45:24.068005Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>visualizacoes</th>\n",
       "      <th>visualizacoes_por_dia</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>858.0</td>\n",
       "      <td>286.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>653.0</td>\n",
       "      <td>59.363636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.066667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1312</th>\n",
       "      <td>647.0</td>\n",
       "      <td>3.634831</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1313</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.011236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1314</th>\n",
       "      <td>214.0</td>\n",
       "      <td>1.188889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1315</th>\n",
       "      <td>520.0</td>\n",
       "      <td>2.872928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1316</th>\n",
       "      <td>540.0</td>\n",
       "      <td>2.967033</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1314 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      visualizacoes  visualizacoes_por_dia\n",
       "0             858.0             286.000000\n",
       "1               2.0               0.285714\n",
       "2               1.0               0.100000\n",
       "3             653.0              59.363636\n",
       "4               2.0               0.066667\n",
       "...             ...                    ...\n",
       "1312          647.0               3.634831\n",
       "1313            2.0               0.011236\n",
       "1314          214.0               1.188889\n",
       "1315          520.0               2.872928\n",
       "1316          540.0               2.967033\n",
       "\n",
       "[1314 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features = df[['visualizacoes', 'visualizacoes_por_dia']]\n",
    "features = features.astype(float)\n",
    "y = df['label'].copy()\n",
    "features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d2ce91b4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:24.826263Z",
     "start_time": "2023-04-20T00:45:24.811065Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((869, 2), (445, 2), (869,), (445,))"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_treino = (df['data_upload'] < '2023-03-01')\n",
    "data_teste = (df['data_upload'] >= '2023-03-01')\n",
    "\n",
    "X_train, X_test = features[data_treino], features[data_teste] \n",
    "y_train, y_test = y[data_treino], y[data_teste] \n",
    "X_train.shape, X_test.shape, y_train.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5b2e5243",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:25.618760Z",
     "start_time": "2023-04-20T00:45:25.503672Z"
    }
   },
   "outputs": [],
   "source": [
    "# TfidfVectorizer -> palavras que aparecerem pouco em muitos vídeos, mas aparecerem\n",
    "# muito em um vídeo, vão ter um peso maior. Palavras que aparecem muito em muitos vídeos\n",
    "# terão menos peso\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "title_train = df[data_treino]['titulo']\n",
    "title_test = df[data_teste]['titulo']\n",
    "\n",
    "#min_df: número mínimo para que uma palavra precisa aparecer nos dados para que essa palavra vire uma coluna. Ex.: machine tem que aparecer em no mínimo dois vídeos para poder virar uma coluna de palavra\n",
    "title_vec = TfidfVectorizer(min_df=2, ngram_range=(1, 5)) \n",
    "title_bow_train = title_vec.fit_transform(title_train)\n",
    "title_bow_test = title_vec.transform(title_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "41593bcc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:26.308938Z",
     "start_time": "2023-04-20T00:45:26.303671Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(869, 2165)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title_bow_train.shape #linhas e colunas, logo achou 188 palavras que repetem 2 ou mais vezes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "08c9f4b5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:26.932451Z",
     "start_time": "2023-04-20T00:45:26.923184Z"
    }
   },
   "outputs": [],
   "source": [
    "from scipy.sparse import hstack, vstack\n",
    "\n",
    "X_train_wtitle = hstack([X_train, title_bow_train])\n",
    "X_test_wtitle = hstack([X_test, title_bow_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a9f52de2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:27.655935Z",
     "start_time": "2023-04-20T00:45:27.647823Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((869, 2167), (445, 2167))"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_wtitle.shape, X_test_wtitle.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fba0e583",
   "metadata": {},
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "55dc852b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:30.743847Z",
     "start_time": "2023-04-20T00:45:28.828556Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(class_weight='balanced', n_estimators=1000, n_jobs=6,\n",
       "                       random_state=0)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# min_samples_leaf = quantos exemplos pra vc precisar ter para criar um novo nó\n",
    "random_forest = RandomForestClassifier(n_estimators=1000, random_state=0, \n",
    "                                       min_samples_leaf=1, class_weight='balanced',\n",
    "                                       n_jobs=6)\n",
    "random_forest.fit(X_train_wtitle, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b7ccba17",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:32.389938Z",
     "start_time": "2023-04-20T00:45:32.157744Z"
    }
   },
   "outputs": [],
   "source": [
    "probabilidade = random_forest.predict_proba(X_test_wtitle)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2a5dad25",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:33.154248Z",
     "start_time": "2023-04-20T00:45:33.145631Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.42419901378303854"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_precision_score(y_test, probabilidade)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9ffa8207",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:33.786665Z",
     "start_time": "2023-04-20T00:45:33.778524Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.877097030151893"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_test, probabilidade)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbf09662",
   "metadata": {},
   "source": [
    "|Modelo| Tipo de modelo| Precision | ROC AUC| \n",
    "|------|---------------|-----------|--------|\n",
    "|modelo 1 (Baseline)| Decision Tree|0.27195362537631207| 0.4626000842815002|\n",
    "|modelo 2 | Random Forest| 0.4190096457477138| 0.6517067003792667| \n",
    "|modelo 3 (aumento teste)| Random Forest| 0.3840157528914409| 0.6327519379844962|\n",
    "|modelo 3 (aumento treino)| Random Forest| 0.46846074116867686| 0.6336915297092287|\n",
    "|modelo 4 (normal)| Random Forest| 0.42419901378303854 | 0.877097030151893|"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c27af99a",
   "metadata": {},
   "source": [
    "## Bayesian Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "154c99ff",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:45:35.740906Z",
     "start_time": "2023-04-20T00:45:35.726843Z"
    }
   },
   "outputs": [],
   "source": [
    "def tune_rf(params):\n",
    "    print(params)\n",
    "    max_depth = params[0]\n",
    "    max_features = params[1]\n",
    "    min_samples_leaf = params[2]\n",
    "    min_samples_split = params[3]\n",
    "    n_estimators = params[4]\n",
    "    min_df = params[5]\n",
    "    ngram_range = (1, params[6])\n",
    "    \n",
    "    title_vec = TfidfVectorizer(min_df=min_df, ngram_range=ngram_range)\n",
    "    title_bow_train = title_vec.fit_transform(title_train)\n",
    "    title_bow_test = title_vec.transform(title_test)\n",
    "    \n",
    "    X_train_wtitle = hstack([X_train, title_bow_train])\n",
    "    X_test_wtitle = hstack([X_test, title_bow_test])\n",
    "    \n",
    "    random_forest = RandomForestClassifier(max_depth=max_depth,\n",
    "                                           max_features=max_features, \n",
    "                                           min_samples_leaf=min_samples_leaf,\n",
    "                                           min_samples_split=min_samples_split, \n",
    "                                           n_estimators=n_estimators,\n",
    "                                           class_weight=\"balanced\", n_jobs=6,\n",
    "                                           random_state=160745)\n",
    "    \n",
    "    random_forest.fit(X_train_wtitle, y_train)\n",
    "    probabilidade = random_forest.predict_proba(X_test_wtitle)[:, 1]\n",
    "    print(roc_auc_score(y_test, probabilidade))\n",
    "    \n",
    "    return -average_precision_score(y_test, probabilidade)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "de8b5fcf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:46:27.774225Z",
     "start_time": "2023-04-20T00:45:39.332706Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration No: 1 started. Evaluating function at random point.\n",
      "[868, 'sqrt', 5, 2, 834, 3, 1]\n",
      "0.8499206529131716\n",
      "Iteration No: 1 ended. Evaluation done at random point.\n",
      "Time taken: 1.6217\n",
      "Function value obtained: -0.3894\n",
      "Current minimum: -0.3894\n",
      "Iteration No: 2 started. Evaluating function at random point.\n",
      "[182, 'auto', 2, 11, 804, 3, 4]\n",
      "0.8747449557923372\n",
      "Iteration No: 2 ended. Evaluation done at random point.\n",
      "Time taken: 1.5879\n",
      "Function value obtained: -0.3851\n",
      "Current minimum: -0.3894\n",
      "Iteration No: 3 started. Evaluating function at random point.\n",
      "[159, 'auto', 5, 7, 1695, 4, 4]\n",
      "0.8622761278621629\n",
      "Iteration No: 3 ended. Evaluation done at random point.\n",
      "Time taken: 3.2371\n",
      "Function value obtained: -0.3856\n",
      "Current minimum: -0.3894\n",
      "Iteration No: 4 started. Evaluating function at random point.\n",
      "[864, 'log2', 8, 13, 1486, 3, 1]\n",
      "0.8492972115166628\n",
      "Iteration No: 4 ended. Evaluation done at random point.\n",
      "Time taken: 2.7850\n",
      "Function value obtained: -0.3841\n",
      "Current minimum: -0.3894\n",
      "Iteration No: 5 started. Evaluating function at random point.\n",
      "[402, 'sqrt', 2, 11, 1852, 5, 2]\n",
      "0.8526411244615734\n",
      "Iteration No: 5 ended. Evaluation done at random point.\n",
      "Time taken: 3.7622\n",
      "Function value obtained: -0.3430\n",
      "Current minimum: -0.3894\n",
      "Iteration No: 6 started. Evaluating function at random point.\n",
      "[234, 'auto', 3, 6, 1182, 4, 1]\n",
      "0.8544547721605078\n",
      "Iteration No: 6 ended. Evaluation done at random point.\n",
      "Time taken: 2.3110\n",
      "Function value obtained: -0.3969\n",
      "Current minimum: -0.3969\n",
      "Iteration No: 7 started. Evaluating function at random point.\n",
      "[1000, 'log2', 10, 20, 588, 2, 5]\n",
      "0.8444797098163681\n",
      "Iteration No: 7 ended. Evaluation done at random point.\n",
      "Time taken: 1.1611\n",
      "Function value obtained: -0.3494\n",
      "Current minimum: -0.3969\n",
      "Iteration No: 8 started. Evaluating function at random point.\n",
      "[366, 'auto', 3, 15, 1594, 3, 2]\n",
      "0.8632963046928134\n",
      "Iteration No: 8 ended. Evaluation done at random point.\n",
      "Time taken: 3.2449\n",
      "Function value obtained: -0.4016\n",
      "Current minimum: -0.4016\n",
      "Iteration No: 9 started. Evaluating function at random point.\n",
      "[938, 'auto', 7, 8, 1298, 1, 2]\n",
      "0.8778338245295851\n",
      "Iteration No: 9 ended. Evaluation done at random point.\n",
      "Time taken: 2.5812\n",
      "Function value obtained: -0.3982\n",
      "Current minimum: -0.4016\n",
      "Iteration No: 10 started. Evaluating function at random point.\n",
      "[922, 'sqrt', 3, 13, 139, 1, 3]\n",
      "0.8858535479483111\n",
      "Iteration No: 10 ended. Evaluation done at random point.\n",
      "Time taken: 0.4949\n",
      "Function value obtained: -0.4348\n",
      "Current minimum: -0.4348\n",
      "Iteration No: 11 started. Searching for the next optimal point.\n",
      "[857, 'sqrt', 1, 16, 1126, 2, 2]\n",
      "0.8777488097936976\n",
      "Iteration No: 11 ended. Search finished for the next optimal point.\n",
      "Time taken: 2.4440\n",
      "Function value obtained: -0.4482\n",
      "Current minimum: -0.4482\n",
      "Iteration No: 12 started. Searching for the next optimal point.\n",
      "[741, 'sqrt', 1, 13, 984, 1, 2]\n",
      "0.8902176377238722\n",
      "Iteration No: 12 ended. Search finished for the next optimal point.\n",
      "Time taken: 2.2631\n",
      "Function value obtained: -0.4671\n",
      "Current minimum: -0.4671\n",
      "Iteration No: 13 started. Searching for the next optimal point.\n",
      "[867, 'sqrt', 2, 18, 1882, 1, 2]\n",
      "0.8879505781002041\n",
      "Iteration No: 13 ended. Search finished for the next optimal point.\n",
      "Time taken: 3.9731\n",
      "Function value obtained: -0.4400\n",
      "Current minimum: -0.4671\n",
      "Iteration No: 14 started. Searching for the next optimal point.\n",
      "[795, 'sqrt', 1, 17, 672, 1, 3]\n",
      "0.8880639310813874\n",
      "Iteration No: 14 ended. Search finished for the next optimal point.\n",
      "Time taken: 1.6454\n",
      "Function value obtained: -0.4609\n",
      "Current minimum: -0.4671\n",
      "Iteration No: 15 started. Searching for the next optimal point.\n",
      "[51, 'sqrt', 1, 8, 993, 1, 3]\n",
      "0.8925980503287236\n",
      "Iteration No: 15 ended. Search finished for the next optimal point.\n",
      "Time taken: 2.3801\n",
      "Function value obtained: -0.4759\n",
      "Current minimum: -0.4759\n",
      "Iteration No: 16 started. Searching for the next optimal point.\n",
      "[234, 'sqrt', 1, 2, 637, 1, 2]\n",
      "0.8882056223078666\n",
      "Iteration No: 16 ended. Search finished for the next optimal point.\n",
      "Time taken: 1.6926\n",
      "Function value obtained: -0.4634\n",
      "Current minimum: -0.4759\n",
      "Iteration No: 17 started. Searching for the next optimal point.\n",
      "[205, 'sqrt', 1, 6, 982, 2, 4]\n",
      "0.8825663114939922\n",
      "Iteration No: 17 ended. Search finished for the next optimal point.\n",
      "Time taken: 2.3465\n",
      "Function value obtained: -0.4594\n",
      "Current minimum: -0.4759\n",
      "Iteration No: 18 started. Searching for the next optimal point.\n",
      "[207, 'sqrt', 1, 5, 583, 2, 4]\n",
      "0.8840399002493765\n",
      "Iteration No: 18 ended. Search finished for the next optimal point.\n",
      "Time taken: 1.5150\n",
      "Function value obtained: -0.4654\n",
      "Current minimum: -0.4759\n",
      "Iteration No: 19 started. Searching for the next optimal point.\n",
      "[68, 'sqrt', 1, 7, 1682, 5, 2]\n",
      "0.8532078893674904\n",
      "Iteration No: 19 ended. Search finished for the next optimal point.\n",
      "Time taken: 3.6682\n",
      "Function value obtained: -0.3330\n",
      "Current minimum: -0.4759\n",
      "Iteration No: 20 started. Searching for the next optimal point.\n",
      "[44, 'sqrt', 1, 11, 1688, 2, 5]\n",
      "0.8827930174563591\n",
      "Iteration No: 20 ended. Search finished for the next optimal point.\n",
      "Time taken: 3.7145\n",
      "Function value obtained: -0.4418\n",
      "Current minimum: -0.4759\n"
     ]
    }
   ],
   "source": [
    "space = [(10, 1000), # max_depth\n",
    "          ('auto', 'sqrt', 'log2'), # max_features\n",
    "          (1, 10), # min_samples_leaf\n",
    "          (2, 20), # min_samples_split\n",
    "          (100, 2000), # n_estimators\n",
    "          (1,5), # min_df\n",
    "          (1,5)] # ngram_range\n",
    "\n",
    "resultado = forest_minimize(tune_rf, space, random_state=160745,\n",
    "                            n_random_starts=10, n_calls=20, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b284ddbd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:46:45.723733Z",
     "start_time": "2023-04-20T00:46:45.717145Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[51, 'sqrt', 1, 8, 993, 1, 3]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resultado.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "46123f39",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:46:52.240954Z",
     "start_time": "2023-04-20T00:46:50.181948Z"
    },
    "code_folding": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8925980503287236 0.47588711602407213\n"
     ]
    }
   ],
   "source": [
    "title_vec = TfidfVectorizer(min_df=1, ngram_range=(1, 3))\n",
    "title_bow_train = title_vec.fit_transform(title_train)\n",
    "title_bow_test = title_vec.transform(title_test)\n",
    "\n",
    "X_train_wtitle = hstack([X_train, title_bow_train])\n",
    "X_test_wtitle = hstack([X_test, title_bow_test])\n",
    "\n",
    "random_forest = RandomForestClassifier(max_depth=51, \n",
    "                                       max_features='sqrt', \n",
    "                                       min_samples_leaf=1,\n",
    "                                       min_samples_split=8, \n",
    "                                       n_estimators=993,\n",
    "                                       class_weight=\"balanced\", n_jobs=6,\n",
    "                                       random_state=160745)\n",
    "\n",
    "random_forest.fit(X_train_wtitle, y_train)\n",
    "probabilidade = random_forest.predict_proba(X_test_wtitle)[:, 1]\n",
    "print(roc_auc_score(y_test, probabilidade), average_precision_score(y_test, probabilidade))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a61d445e",
   "metadata": {},
   "source": [
    "|Modelo| Tipo de modelo| Precision | ROC AUC| Parâmetros\n",
    "|------|---------------|-----------|--------|-----------|\n",
    "|modelo 1 (Baseline)| Decision Tree|0.27195362537631207| 0.4626000842815002|\n",
    "|modelo 2 | Random Forest| 0.4190096457477138| 0.6517067003792667| \n",
    "|modelo 3 (aumento teste)| Random Forest| 0.3840157528914409| 0.6327519379844962|\n",
    "|modelo 3 (aumento treino)| Random Forest| 0.46846074116867686| 0.6336915297092287|\n",
    "|modelo 4 (normal)| Random Forest| 0.42419901378303854 | 0.877097030151893|\n",
    "|modelo 4 (tunado)| Random Forest| 0.47588711602407213 | 0.8925980503287236| [max_depth:51, max_features:'sqrt', min_samples_leaf:1, min_samples_split:8, n_estimators:993, min_df:1, ngram_range:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "58b1521e",
   "metadata": {},
   "source": [
    "# LightGBM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7eb99a01",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:47:00.264417Z",
     "start_time": "2023-04-20T00:47:00.109717Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LGBMClassifier(class_weight='balanced', n_jobs=6, random_state=0)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lgbm = LGBMClassifier(random_state=0, class_weight='balanced', n_jobs=6)\n",
    "lgbm.fit(X_train_wtitle, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "15ba8990",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:47:01.054814Z",
     "start_time": "2023-04-20T00:47:01.046190Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    }
   ],
   "source": [
    "probabilidade = lgbm.predict_proba(X_test_wtitle)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c5fe2db9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:47:01.925585Z",
     "start_time": "2023-04-20T00:47:01.910615Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.3900304192098481, 0.8610292450691452)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_precision_score(y_test, probabilidade), roc_auc_score(y_test, probabilidade)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03202296",
   "metadata": {},
   "source": [
    "|Modelo| Tipo de modelo| Precision | ROC AUC| Parâmetros\n",
    "|------|---------------|-----------|--------|-----------|\n",
    "|modelo 1 (Baseline)| Decision Tree|0.27195362537631207| 0.4626000842815002|\n",
    "|modelo 2 | Random Forest| 0.4190096457477138| 0.6517067003792667| \n",
    "|modelo 3 (aumento teste)| Random Forest| 0.3840157528914409| 0.6327519379844962|\n",
    "|modelo 3 (aumento treino)| Random Forest| 0.46846074116867686| 0.6336915297092287|\n",
    "|modelo 4 (normal)| Random Forest| 0.42419901378303854 | 0.877097030151893|\n",
    "|modelo 4 (tunado)| Random Forest| 0.47588711602407213 | 0.8925980503287236| [max_depth:51, max_features:'sqrt', min_samples_leaf:1, min_samples_split:8, n_estimators:993, min_df:1, ngram_range:3]\n",
    "|modelo 4 (normal)| LightGBM| 0.3900304192098481 | 0.8610292450691452|"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b1805d3",
   "metadata": {},
   "source": [
    "## Bayesian Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "95303707",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:47:04.681182Z",
     "start_time": "2023-04-20T00:47:04.671096Z"
    }
   },
   "outputs": [],
   "source": [
    "def tune_lgbm(params):\n",
    "    print(params)\n",
    "    lr = params[0]\n",
    "    max_depth = params[1]\n",
    "    min_child_samples = params[2]\n",
    "    subsample = params[3]\n",
    "    colsample_bytree = params[4]\n",
    "    n_estimators = params[5]\n",
    "    min_df = params[6]\n",
    "    ngram_range = (1, params[7])\n",
    "    \n",
    "    title_vec = TfidfVectorizer(min_df=min_df, ngram_range=ngram_range)\n",
    "    title_bow_train = title_vec.fit_transform(title_train)\n",
    "    title_bow_test = title_vec.transform(title_test)\n",
    "    \n",
    "    X_train_wtitle = hstack([X_train, title_bow_train])\n",
    "    X_test_wtitle = hstack([X_test, title_bow_test])\n",
    "    \n",
    "    lgbm = LGBMClassifier(learning_rate=lr, num_leaves=2 ** max_depth,\n",
    "                         max_depth=max_depth, min_child_samples=min_child_samples,\n",
    "                         subsample=subsample, colsample_bytree=colsample_bytree, \n",
    "                         bagging_freq=1, n_estimators=n_estimators, random_state=0,\n",
    "                         class_weight=\"balanced\", n_jobs=6)\n",
    "    lgbm.fit(X_train_wtitle, y_train)\n",
    "    probabilidade = lgbm.predict_proba(X_test_wtitle)[:, 1]\n",
    "    print(roc_auc_score(y_test, probabilidade))\n",
    "    \n",
    "    return -average_precision_score(y_test, probabilidade)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3b1955ad",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:47:30.736563Z",
     "start_time": "2023-04-20T00:47:12.048890Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration No: 1 started. Evaluating function at random point.\n",
      "[0.009944912110647982, 5, 1, 0.4677107511929402, 0.49263223036174764, 182, 3, 1]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n",
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8254930854681478\n",
      "Iteration No: 1 ended. Evaluation done at random point.\n",
      "Time taken: 0.3187\n",
      "Function value obtained: -0.3261\n",
      "Current minimum: -0.3261\n",
      "Iteration No: 2 started. Evaluating function at random point.\n",
      "[0.053887464791860025, 1, 15, 0.7437489153990157, 0.8675167974293533, 459, 3, 4]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n",
      "0.8336828383586489\n",
      "Iteration No: 2 ended. Evaluation done at random point.\n",
      "Time taken: 0.1204\n",
      "Function value obtained: -0.3436\n",
      "Current minimum: -0.3436\n",
      "Iteration No: 3 started. Evaluating function at random point.\n",
      "[0.004151454520895999, 6, 20, 0.8682075103820793, 0.9491436163200662, 321, 4, 3]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8660167762412152\n",
      "Iteration No: 3 ended. Evaluation done at random point.\n",
      "Time taken: 0.2140\n",
      "Function value obtained: -0.3665\n",
      "Current minimum: -0.3665\n",
      "Iteration No: 4 started. Evaluating function at random point.\n",
      "[0.0014099928811969545, 9, 9, 0.6502182010234373, 0.6866210554187129, 979, 1, 5]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8971321695760599\n",
      "Iteration No: 4 ended. Evaluation done at random point.\n",
      "Time taken: 1.1606\n",
      "Function value obtained: -0.5065\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 5 started. Evaluating function at random point.\n",
      "[0.0012872056698236922, 6, 8, 0.3448850990367951, 0.371803112561826, 703, 4, 1]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8530378598957152\n",
      "Iteration No: 5 ended. Evaluation done at random point.\n",
      "Time taken: 0.5523\n",
      "Function value obtained: -0.4159\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 6 started. Evaluating function at random point.\n",
      "[0.003567949451535685, 10, 19, 0.7232951768944309, 0.7298538828427115, 849, 4, 3]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8778054862842892\n",
      "Iteration No: 6 ended. Evaluation done at random point.\n",
      "Time taken: 0.7628\n",
      "Function value obtained: -0.4238\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 7 started. Evaluating function at random point.\n",
      "[0.014828577273549474, 7, 1, 0.18428087097824575, 0.3261556557915816, 184, 1, 2]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.861085921559737\n",
      "Iteration No: 7 ended. Evaluation done at random point.\n",
      "Time taken: 0.9328\n",
      "Function value obtained: -0.4665\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 8 started. Evaluating function at random point.\n",
      "[0.0015212976972079912, 3, 12, 0.44234694306528044, 0.399351303640462, 182, 3, 5]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n",
      "0.8739231466787577\n",
      "Iteration No: 8 ended. Evaluation done at random point.\n",
      "Time taken: 0.2059\n",
      "Function value obtained: -0.4110\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 9 started. Evaluating function at random point.\n",
      "[0.01946212855369041, 9, 18, 0.5235636153223084, 0.6728679300083596, 657, 4, 5]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8463500340058944\n",
      "Iteration No: 9 ended. Evaluation done at random point.\n",
      "Time taken: 0.5021\n",
      "Function value obtained: -0.3424\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 10 started. Evaluating function at random point.\n",
      "[0.0012116790683302117, 3, 2, 0.06616307483844217, 0.23025600705315752, 587, 2, 5]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8577986851054182\n",
      "Iteration No: 10 ended. Evaluation done at random point.\n",
      "Time taken: 1.5761\n",
      "Function value obtained: -0.3864\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 11 started. Searching for the next optimal point.\n",
      "[0.05696329143679483, 10, 10, 0.7704339000317592, 0.5363769745994437, 818, 2, 2]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8034459306279755\n",
      "Iteration No: 11 ended. Search finished for the next optimal point.\n",
      "Time taken: 1.5818\n",
      "Function value obtained: -0.3845\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 12 started. Searching for the next optimal point.\n",
      "[0.0011992657133966242, 8, 6, 0.9129013695364318, 0.6669627457969953, 982, 2, 5]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8354681478122875\n",
      "Iteration No: 12 ended. Search finished for the next optimal point.\n",
      "Time taken: 2.4070\n",
      "Function value obtained: -0.3589\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 13 started. Searching for the next optimal point.\n",
      "[0.0011181590238459151, 9, 15, 0.5149880123416892, 0.645885726051927, 836, 2, 5]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8800725459079574\n",
      "Iteration No: 13 ended. Search finished for the next optimal point.\n",
      "Time taken: 0.8401\n",
      "Function value obtained: -0.4507\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 14 started. Searching for the next optimal point.\n",
      "[0.0025076342055892287, 10, 11, 0.22866269936746775, 0.7488403059207158, 882, 2, 5]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8630695987304466\n",
      "Iteration No: 14 ended. Search finished for the next optimal point.\n",
      "Time taken: 0.9337\n",
      "Function value obtained: -0.4002\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 15 started. Searching for the next optimal point.\n",
      "[0.0014331286968287877, 9, 9, 0.5805040445517531, 0.7546293525384392, 410, 2, 5]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8780321922466561\n",
      "Iteration No: 15 ended. Search finished for the next optimal point.\n",
      "Time taken: 0.6533\n",
      "Function value obtained: -0.4477\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 16 started. Searching for the next optimal point.\n",
      "[0.00200534010861793, 9, 9, 0.6410348508642206, 0.9779588977174413, 922, 2, 4]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8774087508501474\n",
      "Iteration No: 16 ended. Search finished for the next optimal point.\n",
      "Time taken: 1.2833\n",
      "Function value obtained: -0.4191\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 17 started. Searching for the next optimal point.\n",
      "[0.0010828743643737323, 9, 8, 0.7000271364885504, 0.8498309079688997, 933, 1, 5]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8881206075719792\n",
      "Iteration No: 17 ended. Search finished for the next optimal point.\n",
      "Time taken: 1.2988\n",
      "Function value obtained: -0.4891\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 18 started. Searching for the next optimal point.\n",
      "[0.04701635352606865, 6, 14, 0.31996393654102856, 0.5045672957333033, 397, 1, 2]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8245862616186805\n",
      "Iteration No: 18 ended. Search finished for the next optimal point.\n",
      "Time taken: 0.5321\n",
      "Function value obtained: -0.3554\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 19 started. Searching for the next optimal point.\n",
      "[0.008663929625020213, 9, 6, 0.9727910200295122, 0.8353148654941072, 909, 1, 5]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8314441169802766\n",
      "Iteration No: 19 ended. Search finished for the next optimal point.\n",
      "Time taken: 1.3587\n",
      "Function value obtained: -0.4196\n",
      "Current minimum: -0.5065\n",
      "Iteration No: 20 started. Searching for the next optimal point.\n",
      "[0.0012948667679063663, 8, 6, 0.9805865967329576, 0.7187568369865114, 870, 1, 5]\n",
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8630695987304466\n",
      "Iteration No: 20 ended. Search finished for the next optimal point.\n",
      "Time taken: 1.4412\n",
      "Function value obtained: -0.4191\n",
      "Current minimum: -0.5065\n"
     ]
    }
   ],
   "source": [
    "space = [(1e-3, 1e-1, 'log-uniform'), #lr\n",
    "       (1, 10), #max_depth\n",
    "       (1, 20), #min_child_samples\n",
    "       (0.05, 1.), #subsample\n",
    "       (0.05, 1.), # colsample_bytree  \n",
    "       (10, 1000), #n_estimators\n",
    "       (1, 5), # min_df\n",
    "       (1, 5)] # ngram_range\n",
    "\n",
    "resultado = forest_minimize(tune_lgbm, space, random_state=160745,\n",
    "                            n_random_starts=10, n_calls=20, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "595e4494",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:49:42.637578Z",
     "start_time": "2023-04-20T00:49:42.629342Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.0014099928811969545,\n",
       " 9,\n",
       " 9,\n",
       " 0.6502182010234373,\n",
       " 0.6866210554187129,\n",
       " 979,\n",
       " 1,\n",
       " 5]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resultado.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3dd11d61",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:49:46.882477Z",
     "start_time": "2023-04-20T00:49:45.737902Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] bagging_freq is set=1, subsample_freq=0 will be ignored. Current value: bagging_freq=1\n",
      "0.5064733173326582 0.8971321695760599\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/lightgbm/basic.py:597: UserWarning: Converting data to scipy sparse matrix.\n",
      "  warnings.warn('Converting data to scipy sparse matrix.')\n"
     ]
    }
   ],
   "source": [
    "title_vec = TfidfVectorizer(min_df=1, ngram_range=(1, 5))\n",
    "title_bow_train = title_vec.fit_transform(title_train)\n",
    "title_bow_test = title_vec.transform(title_test)\n",
    "\n",
    "X_train_wtitle = hstack([X_train, title_bow_train])\n",
    "X_test_wtitle = hstack([X_test, title_bow_test])\n",
    "\n",
    "lgbm = LGBMClassifier(learning_rate=0.0014099928811969545, num_leaves=2 ** 9,\n",
    "                     max_depth=9, min_child_samples=9,\n",
    "                     subsample=0.6502182010234373, colsample_bytree= 0.6866210554187129, \n",
    "                     bagging_freq=1, n_estimators=979, random_state=0,\n",
    "                     class_weight=\"balanced\", n_jobs=6)\n",
    "lgbm.fit(X_train_wtitle, y_train)\n",
    "probabilidade = lgbm.predict_proba(X_test_wtitle)[:, 1]\n",
    "print(average_precision_score(y_test, probabilidade), roc_auc_score(y_test, probabilidade))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9add064b",
   "metadata": {},
   "source": [
    "|Modelo| Tipo de modelo| Precision | ROC AUC| Parâmetros\n",
    "|------|---------------|-----------|--------|-----------|\n",
    "|modelo 1 (Baseline)| Decision Tree|0.27195362537631207| 0.4626000842815002|\n",
    "|modelo 2 | Random Forest| 0.4190096457477138| 0.6517067003792667| \n",
    "|modelo 3 (aumento teste)| Random Forest| 0.3840157528914409| 0.6327519379844962|\n",
    "|modelo 3 (aumento treino)| Random Forest| 0.46846074116867686| 0.6336915297092287|\n",
    "|modelo 4 (normal)| Random Forest| 0.42419901378303854 | 0.877097030151893|\n",
    "|modelo 4 (tunado)| Random Forest| 0.47588711602407213 | 0.8925980503287236| [max_depth:51, max_features:'sqrt', min_samples_leaf:1, min_samples_split:8, n_estimators:993, min_df:1, ngram_range:3]\n",
    "|modelo 4 (normal)| LightGBM| 0.3900304192098481 | 0.8610292450691452|\n",
    "|modelo 4 (tunado)| LightGBM| 0.5064733173326582 | 0.8971321695760599| [lr:0.0014099928811969545, max_depth:9, min_child_samples:9, subsample:0.6502182010234373, colsample_bytree:0.6866210554187129, n_estimators:979,min_df:1, ngram_range:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fdeae93f",
   "metadata": {},
   "source": [
    "# Regressão Logistíca"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "ec08ae7c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:49:55.919482Z",
     "start_time": "2023-04-20T00:49:55.915346Z"
    }
   },
   "outputs": [],
   "source": [
    "# maxabsclares: pega o maior valor da coluna, pega seu valor absoluto e dividi todos os outros valores por ele\n",
    "from sklearn.preprocessing import MaxAbsScaler, StandardScaler\n",
    "from scipy.sparse import csr_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8e3ddd87",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:49:56.739296Z",
     "start_time": "2023-04-20T00:49:56.716092Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/sklearn/utils/validation.py:593: FutureWarning: np.matrix usage is deprecated in 1.0 and will raise a TypeError in 1.2. Please convert to a numpy array with np.asarray. For more information see: https://numpy.org/doc/stable/reference/generated/numpy.matrix.html\n",
      "  warnings.warn(\n",
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/sklearn/utils/validation.py:593: FutureWarning: np.matrix usage is deprecated in 1.0 and will raise a TypeError in 1.2. Please convert to a numpy array with np.asarray. For more information see: https://numpy.org/doc/stable/reference/generated/numpy.matrix.html\n",
      "  warnings.warn(\n",
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/scipy/sparse/_index.py:146: SparseEfficiencyWarning: Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.\n",
      "  self._set_arrayXarray(i, j, x)\n",
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/sklearn/utils/validation.py:593: FutureWarning: np.matrix usage is deprecated in 1.0 and will raise a TypeError in 1.2. Please convert to a numpy array with np.asarray. For more information see: https://numpy.org/doc/stable/reference/generated/numpy.matrix.html\n",
      "  warnings.warn(\n",
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/sklearn/utils/validation.py:593: FutureWarning: np.matrix usage is deprecated in 1.0 and will raise a TypeError in 1.2. Please convert to a numpy array with np.asarray. For more information see: https://numpy.org/doc/stable/reference/generated/numpy.matrix.html\n",
      "  warnings.warn(\n",
      "/home/fhfraga/anaconda3/lib/python3.9/site-packages/scipy/sparse/_index.py:146: SparseEfficiencyWarning: Changing the sparsity structure of a csr_matrix is expensive. lil_matrix is more efficient.\n",
      "  self._set_arrayXarray(i, j, x)\n"
     ]
    }
   ],
   "source": [
    "X_train_wtitle2 = csr_matrix(X_train_wtitle.copy())\n",
    "X_test_wtitle2 = csr_matrix(X_test_wtitle.copy())\n",
    "\n",
    "scaler = StandardScaler()\n",
    "max_abs_scaler = MaxAbsScaler()\n",
    "\n",
    "X_train_wtitle2[:, :2] = scaler.fit_transform(X_train_wtitle2[:, :2].todense())\n",
    "X_test_wtitle2[:, :2] = scaler.fit_transform(X_test_wtitle2[:, :2].todense())\n",
    "\n",
    "#X_train_wtitle2 = max_abs_scaler.fit_transform(X_train_wtitle2)\n",
    "#X_test_wtitle2= max_abs_scaler.fit_transform(X_test_wtitle2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "abc76a7f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:49:58.862493Z",
     "start_time": "2023-04-20T00:49:57.508456Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=10, n_jobs=6, random_state=0)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logistic_regression = LogisticRegression(C=10, n_jobs=6, random_state=0)\n",
    "logistic_regression.fit(X_train_wtitle2, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "295e9d72",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:49:59.697557Z",
     "start_time": "2023-04-20T00:49:59.691852Z"
    }
   },
   "outputs": [],
   "source": [
    "probabilidade = logistic_regression.predict_proba(X_test_wtitle2)[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b19404b9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-04-20T00:50:00.425348Z",
     "start_time": "2023-04-20T00:50:00.412434Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.45449846483463946 0.8893674903649966\n"
     ]
    }
   ],
   "source": [
    "print(average_precision_score(y_test, probabilidade), roc_auc_score(y_test, probabilidade))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ac327a6",
   "metadata": {},
   "source": [
    "|Modelo| Tipo de modelo| Precision | ROC AUC| Parâmetros\n",
    "|------|---------------|-----------|--------|-----------|\n",
    "|modelo 1 (Baseline)| Decision Tree|0.27195362537631207| 0.4626000842815002|\n",
    "|modelo 2 | Random Forest| 0.4190096457477138| 0.6517067003792667| \n",
    "|modelo 3 (aumento teste)| Random Forest| 0.3840157528914409| 0.6327519379844962|\n",
    "|modelo 3 (aumento treino)| Random Forest| 0.46846074116867686| 0.6336915297092287|\n",
    "|modelo 4 (normal)| Random Forest| 0.42419901378303854 | 0.877097030151893|\n",
    "|modelo 4 (tunado)| Random Forest| 0.47588711602407213 | 0.8925980503287236| [max_depth:51, max_features:'sqrt', min_samples_leaf:1, min_samples_split:8, n_estimators:993, min_df:1, ngram_range:3]\n",
    "|modelo 4 (normal)| LightGBM| 0.3900304192098481 | 0.8610292450691452|\n",
    "|modelo 4 (tunado)| LightGBM| 0.5064733173326582 | 0.8971321695760599| [lr:0.0014099928811969545, max_depth:9, min_child_samples:9, subsample:0.6502182010234373, colsample_bytree:0.6866210554187129, n_estimators:979,min_df:1, ngram_range:5]\n",
    "|modelo 4 (tunado)| Logistic Regression| 0.45449846483463946 | 0.8893674903649966| [C=10]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1b77a0d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "nbTranslate": {
   "displayLangs": [
    "*"
   ],
   "hotkey": "alt-t",
   "langInMainMenu": true,
   "sourceLang": "en",
   "targetLang": "fr",
   "useGoogleTranslate": true
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}