Automated-essay-scoring-main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "de095e1f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:45:56.991708Z",
     "iopub.status.busy": "2024-06-13T21:45:56.990963Z",
     "iopub.status.idle": "2024-06-13T21:45:57.001628Z",
     "shell.execute_reply": "2024-06-13T21:45:57.000886Z"
    },
    "papermill": {
     "duration": 0.026982,
     "end_time": "2024-06-13T21:45:57.003509",
     "exception": false,
     "start_time": "2024-06-13T21:45:56.976527",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import re\n",
    "\n",
    "INPUT_DIR = Path(\"../input/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6022c059",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:45:57.030613Z",
     "iopub.status.busy": "2024-06-13T21:45:57.030328Z",
     "iopub.status.idle": "2024-06-13T21:46:30.426770Z",
     "shell.execute_reply": "2024-06-13T21:46:30.425738Z"
    },
    "papermill": {
     "duration": 33.412955,
     "end_time": "2024-06-13T21:46:30.429095",
     "exception": false,
     "start_time": "2024-06-13T21:45:57.016140",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install -q /kaggle/input/language-tool-python-2-7-1/language_tool_python-2.7.1-py3-none-any.whl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bbb030dc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:46:30.455681Z",
     "iopub.status.busy": "2024-06-13T21:46:30.455373Z",
     "iopub.status.idle": "2024-06-13T21:46:30.464984Z",
     "shell.execute_reply": "2024-06-13T21:46:30.464151Z"
    },
    "papermill": {
     "duration": 0.025048,
     "end_time": "2024-06-13T21:46:30.466969",
     "exception": false,
     "start_time": "2024-06-13T21:46:30.441921",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/root/.cache/language_tool_python'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import zipfile\n",
    "from zipfile import ZipFile\n",
    "import shutil\n",
    "\n",
    "# create download path\n",
    "def get_language_tool_cache_path():\n",
    "    # Get download path from environment or use default.\n",
    "    download_path = os.environ.get(\n",
    "        'LTP_PATH',\n",
    "        os.path.join(os.path.expanduser(\"~\"), \".cache\", \"language_tool_python\")\n",
    "    )\n",
    "    # Make download path, if it doesn't exist.\n",
    "    os.makedirs(download_path, exist_ok=True)\n",
    "    return download_path\n",
    "\n",
    "lt_path = get_language_tool_cache_path()\n",
    "lt_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5e9bd85a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:46:30.494811Z",
     "iopub.status.busy": "2024-06-13T21:46:30.494508Z",
     "iopub.status.idle": "2024-06-13T21:46:48.136378Z",
     "shell.execute_reply": "2024-06-13T21:46:48.135305Z"
    },
    "papermill": {
     "duration": 17.658096,
     "end_time": "2024-06-13T21:46:48.138750",
     "exception": false,
     "start_time": "2024-06-13T21:46:30.480654",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All files zipped successfully!\n",
      "Extracted all\n",
      "['LanguageTool-5.7']\n"
     ]
    }
   ],
   "source": [
    "def get_all_file_paths(directory):\n",
    "  \n",
    "    # initializing empty file paths list\n",
    "    file_paths = []\n",
    "  \n",
    "    # crawling through directory and subdirectories\n",
    "    for root, directories, files in os.walk(directory):\n",
    "        for filename in files:\n",
    "            # join the two strings in order to form the full filepath.\n",
    "            filepath = os.path.join(root, filename)\n",
    "            file_paths.append(filepath)\n",
    "  \n",
    "    # returning all file paths\n",
    "    return file_paths        \n",
    "\n",
    "def main():\n",
    "    # path to folder which needs to be zipped\n",
    "    directory = '../input/language-tool-python-2-7-1/LanguageTool-5.7/LanguageTool-5.7'\n",
    "  \n",
    "    # calling function to get all file paths in the directory\n",
    "    file_paths = get_all_file_paths(directory)\n",
    "\n",
    "    # writing files to a zipfile\n",
    "    with ZipFile('./lt.zip','w') as zip:\n",
    "        # writing each file one by one\n",
    "        for file in file_paths:\n",
    "            zip.write(file)\n",
    "  \n",
    "    print('All files zipped successfully!')        \n",
    "    \n",
    "main()\n",
    " \n",
    "zip_file = \"./lt.zip\"\n",
    "\n",
    "try:\n",
    "    with zipfile.ZipFile(zip_file) as z:\n",
    "        z.extractall()\n",
    "        print(\"Extracted all\")\n",
    "except:\n",
    "    print(\"Invalid file\")\n",
    "    \n",
    "#move to cache\n",
    "!mv {'./input/language-tool-python-2-7-1/LanguageTool-5.7/LanguageTool-5.7'} {lt_path} \n",
    "print(os.listdir('/root/.cache/language_tool_python/'))\n",
    "\n",
    "#remove files from output\n",
    "\n",
    "shutil.rmtree('./input')\n",
    "os.remove(\"./lt.zip\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "516d7b63",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:46:48.166560Z",
     "iopub.status.busy": "2024-06-13T21:46:48.165730Z",
     "iopub.status.idle": "2024-06-13T21:46:50.254207Z",
     "shell.execute_reply": "2024-06-13T21:46:50.253052Z"
    },
    "papermill": {
     "duration": 2.105383,
     "end_time": "2024-06-13T21:46:50.257337",
     "exception": false,
     "start_time": "2024-06-13T21:46:48.151954",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import language_tool_python\n",
    "tool = language_tool_python.LanguageTool('en-US')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "406f92af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:46:50.295890Z",
     "iopub.status.busy": "2024-06-13T21:46:50.295477Z",
     "iopub.status.idle": "2024-06-13T21:46:54.736204Z",
     "shell.execute_reply": "2024-06-13T21:46:54.735298Z"
    },
    "papermill": {
     "duration": 4.46247,
     "end_time": "2024-06-13T21:46:54.738471",
     "exception": false,
     "start_time": "2024-06-13T21:46:50.276001",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CUDA_AVAILABLE = True\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import gc\n",
    "import pickle\n",
    "import torch\n",
    "\n",
    "\n",
    "_test = pd.read_csv(\"/kaggle/input/learning-agency-lab-automated-essay-scoring-2/test.csv\")\n",
    "ENABLE_DONT_WASTE_YOUR_RUN_TIME = len(_test) < 10\n",
    "\n",
    "if ENABLE_DONT_WASTE_YOUR_RUN_TIME:\n",
    "    import shutil\n",
    "    del _test\n",
    "    gc.collect()\n",
    "\n",
    "import torch\n",
    "\n",
    "CUDA_AVAILABLE = torch.cuda.is_available()\n",
    "print(f\"{CUDA_AVAILABLE = }\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "245471ec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:46:54.767075Z",
     "iopub.status.busy": "2024-06-13T21:46:54.766771Z",
     "iopub.status.idle": "2024-06-13T21:47:08.925925Z",
     "shell.execute_reply": "2024-06-13T21:47:08.925139Z"
    },
    "papermill": {
     "duration": 14.176062,
     "end_time": "2024-06-13T21:47:08.928366",
     "exception": false,
     "start_time": "2024-06-13T21:46:54.752304",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-06-13 21:47:00.740260: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-06-13 21:47:00.740406: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-06-13 21:47:00.848358: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n"
     ]
    }
   ],
   "source": [
    "import xgboost as xgb\n",
    "import pandas as pd \n",
    "from transformers import (\n",
    "    AutoTokenizer, \n",
    "    AutoModelForSequenceClassification, \n",
    "    Trainer, \n",
    "    TrainingArguments, \n",
    "    DataCollatorWithPadding\n",
    ")\n",
    "from datasets import Dataset\n",
    "from glob import glob\n",
    "import gc\n",
    "import torch\n",
    "from scipy.special import softmax\n",
    "\n",
    "MAX_LENGTH = 1024  #1024 \n",
    "TEST_DATA_PATH = \"/kaggle/input/learning-agency-lab-automated-essay-scoring-2/test.csv\"\n",
    "MODEL_PATH = '/kaggle/input/aes2-400-20240419134941/*/*'\n",
    "EVAL_BATCH_SIZE = 1 #1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "173d06aa",
   "metadata": {
    "papermill": {
     "duration": 0.012686,
     "end_time": "2024-06-13T21:47:08.954352",
     "exception": false,
     "start_time": "2024-06-13T21:47:08.941666",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Deberta Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4c2300d2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:47:08.981516Z",
     "iopub.status.busy": "2024-06-13T21:47:08.980953Z",
     "iopub.status.idle": "2024-06-13T21:48:33.995021Z",
     "shell.execute_reply": "2024-06-13T21:48:33.994272Z"
    },
    "papermill": {
     "duration": 85.030221,
     "end_time": "2024-06-13T21:48:33.997360",
     "exception": false,
     "start_time": "2024-06-13T21:47:08.967139",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bff0561369cc4812835a1cd6171e0c0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/3 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py:432: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches', 'even_batches', 'use_seedable_sampler']). Please pass an `accelerate.DataLoaderConfiguration` instead: \n",
      "dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False, even_batches=True, use_seedable_sampler=True)\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "models = glob(MODEL_PATH)\n",
    "tokenizer = AutoTokenizer.from_pretrained(models[0])\n",
    "\n",
    "def tokenize(sample):\n",
    "    return tokenizer(sample['full_text'], max_length=MAX_LENGTH, truncation=True)\n",
    "\n",
    "df_test = pd.read_csv(TEST_DATA_PATH)\n",
    "ds = Dataset.from_pandas(df_test).map(tokenize).remove_columns(['essay_id', 'full_text'])\n",
    "\n",
    "args = TrainingArguments(\n",
    "    \".\", \n",
    "    per_device_eval_batch_size=EVAL_BATCH_SIZE, \n",
    "    report_to=\"none\"\n",
    ")\n",
    "\n",
    "predictions = []\n",
    "for model in models:\n",
    "    model = AutoModelForSequenceClassification.from_pretrained(model)\n",
    "    trainer = Trainer(\n",
    "        model=model, \n",
    "        args=args, \n",
    "        data_collator=DataCollatorWithPadding(tokenizer), \n",
    "        tokenizer=tokenizer\n",
    "    )\n",
    "    \n",
    "    preds = trainer.predict(ds).predictions\n",
    "    predictions.append(softmax(preds, axis=-1))  \n",
    "    del model, trainer\n",
    "    torch.cuda.empty_cache()\n",
    "    gc.collect()\n",
    "    \n",
    "predicted_score = 0.\n",
    "\n",
    "for p in predictions:\n",
    "    predicted_score += p\n",
    "    \n",
    "predicted_score /= len(predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b329588c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:48:34.027356Z",
     "iopub.status.busy": "2024-06-13T21:48:34.027018Z",
     "iopub.status.idle": "2024-06-13T21:48:34.040262Z",
     "shell.execute_reply": "2024-06-13T21:48:34.039409Z"
    },
    "papermill": {
     "duration": 0.030417,
     "end_time": "2024-06-13T21:48:34.042216",
     "exception": false,
     "start_time": "2024-06-13T21:48:34.011799",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay_id</th>\n",
       "      <th>full_text</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000d118</td>\n",
       "      <td>Many people have car where they live. The thin...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000fe60</td>\n",
       "      <td>I am a scientist at NASA that is discussing th...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>001ab80</td>\n",
       "      <td>People always wish they had the same technolog...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  essay_id                                          full_text  score\n",
       "0  000d118  Many people have car where they live. The thin...      3\n",
       "1  000fe60  I am a scientist at NASA that is discussing th...      3\n",
       "2  001ab80  People always wish they had the same technolog...      5"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_test['score'] = predicted_score.argmax(-1) + 1\n",
    "df_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0edd958a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:48:34.071877Z",
     "iopub.status.busy": "2024-06-13T21:48:34.071603Z",
     "iopub.status.idle": "2024-06-13T21:48:34.143389Z",
     "shell.execute_reply": "2024-06-13T21:48:34.142452Z"
    },
    "papermill": {
     "duration": 0.088793,
     "end_time": "2024-06-13T21:48:34.145513",
     "exception": false,
     "start_time": "2024-06-13T21:48:34.056720",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df_test[['essay_id', 'score']].to_csv('submission1.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e3feb602",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:48:34.174867Z",
     "iopub.status.busy": "2024-06-13T21:48:34.174569Z",
     "iopub.status.idle": "2024-06-13T21:49:06.104267Z",
     "shell.execute_reply": "2024-06-13T21:49:06.103139Z"
    },
    "papermill": {
     "duration": 31.960678,
     "end_time": "2024-06-13T21:49:06.120456",
     "exception": false,
     "start_time": "2024-06-13T21:48:34.159778",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: language-tool-python in /opt/conda/lib/python3.10/site-packages (2.7.1)\r\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from language-tool-python) (2.31.0)\r\n",
      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from language-tool-python) (4.66.1)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->language-tool-python) (3.3.2)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->language-tool-python) (3.6)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->language-tool-python) (1.26.18)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->language-tool-python) (2024.2.2)\r\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install language-tool-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0bd3f769",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:49:06.150863Z",
     "iopub.status.busy": "2024-06-13T21:49:06.150503Z",
     "iopub.status.idle": "2024-06-13T21:49:10.733553Z",
     "shell.execute_reply": "2024-06-13T21:49:10.732636Z"
    },
    "papermill": {
     "duration": 4.601146,
     "end_time": "2024-06-13T21:49:10.735929",
     "exception": false,
     "start_time": "2024-06-13T21:49:06.134783",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/dask/dataframe/_pyarrow_compat.py:23: UserWarning: You are using pyarrow version 11.0.0 which is known to be insecure. See https://www.cve.org/CVERecord?id=CVE-2023-47248 for further details. Please upgrade to pyarrow>=14.0.1 or install pyarrow-hotfix to patch your current version.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Importing necessary libraries\n",
    "import gc\n",
    "import lightgbm as lgb\n",
    "from sklearn.ensemble import VotingRegressor\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import re\n",
    "import spacy\n",
    "import language_tool_python\n",
    "import string\n",
    "import random\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "from nltk.corpus import wordnet as wn\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV\n",
    "from sklearn.ensemble import GradientBoostingClassifier,BaggingClassifier\n",
    "from sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer, HashingVectorizer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold\n",
    "from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, f1_score\n",
    "from sklearn.metrics import cohen_kappa_score\n",
    "from lightgbm import log_evaluation, early_stopping\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "import polars as pl\n",
    "import joblib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "beb05858",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:49:10.766614Z",
     "iopub.status.busy": "2024-06-13T21:49:10.766281Z",
     "iopub.status.idle": "2024-06-13T21:49:12.850481Z",
     "shell.execute_reply": "2024-06-13T21:49:12.849615Z"
    },
    "papermill": {
     "duration": 2.101694,
     "end_time": "2024-06-13T21:49:12.852614",
     "exception": false,
     "start_time": "2024-06-13T21:49:10.750920",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (1, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>essay_id</th><th>full_text</th><th>score</th><th>paragraph</th></tr><tr><td>str</td><td>str</td><td>i64</td><td>list[str]</td></tr></thead><tbody><tr><td>&quot;000d118&quot;</td><td>&quot;Many people ha…</td><td>3</td><td>[&quot;Many people have car where they live. The thing they don&#x27;t know is that when you use a car alot of thing can happen like you can get in accidet or the smoke that the car has is bad to breath on if someone is walk but in VAUBAN,Germany they dont have that proble because 70 percent of vauban&#x27;s families do not own cars,and 57 percent sold a car to move there. Street parkig ,driveways and home garages are forbidden on the outskirts of freiburd that near the French and Swiss borders. You probaly won&#x27;t see a car in Vauban&#x27;s streets because they are completely &quot;car free&quot; but If some that lives in VAUBAN that owns a car ownership is allowed,but there are only two places that you can park a large garages at the edge of the development,where a car owner buys a space but it not cheap to buy one they sell the space for you car for $40,000 along with a home. The vauban people completed this in 2006 ,they said that this an example of a growing trend in Europe,The untile states and some where else are suburban life from auto use this is called &quot;smart planning&quot;. The current efforts to drastically reduce greenhouse gas emissions from tailes the passengee cars are responsible for 12 percent of greenhouse gas emissions in Europe and up to 50 percent in some car intensive in the United States. I honeslty think that good idea that they did that is Vaudan because that makes cities denser and better for walking and in VAUBAN there are 5,500 residents within a rectangular square mile. In the artical David Gold berg said that &quot;All of our development since World war 2 has been centered on the cars,and that will have to change&quot; and i think that was very true what David Gold said because alot thing we need cars to do we can go anyway were with out cars beacuse some people are a very lazy to walk to place thats why they alot of people use car and i think that it was a good idea that that they did that in VAUBAN so people can see how we really don&#x27;t need car to go to place from place because we can walk from were we need to go or we can ride bycles with out the use of a car. It good that they are doing that if you thik about your help the earth in way and thats a very good thing to. In the United states ,the Environmental protection Agency is promoting what is called &quot;car reduced&quot;communtunties,and the legislators are starting to act,if cautiously. Maany experts expect pubic transport serving suburbs to play a much larger role in a new six years federal transportation bill to approved this year. In previous bill,80 percent of appropriations have by law gone to highways and only 20 percent to other transports. There many good reason why they should do this.    &quot;]</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (1, 4)\n",
       "┌──────────┬───────────────────────────────────┬───────┬───────────────────────────────────┐\n",
       "│ essay_id ┆ full_text                         ┆ score ┆ paragraph                         │\n",
       "│ ---      ┆ ---                               ┆ ---   ┆ ---                               │\n",
       "│ str      ┆ str                               ┆ i64   ┆ list[str]                         │\n",
       "╞══════════╪═══════════════════════════════════╪═══════╪═══════════════════════════════════╡\n",
       "│ 000d118  ┆ Many people have car where they … ┆ 3     ┆ [\"Many people have car where the… │\n",
       "└──────────┴───────────────────────────────────┴───────┴───────────────────────────────────┘"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "columns = [  \n",
    "    (\n",
    "        pl.col(\"full_text\").str.split(by=\"\\n\\n\").alias(\"paragraph\")\n",
    "    ),\n",
    "]\n",
    "PATH = \"/kaggle/input/learning-agency-lab-automated-essay-scoring-2/\"\n",
    "\n",
    "# Load training and testing sets, while using \\ n \\ n character segmentation to list and renaming to paragraph for full_text data\n",
    "train = pl.read_csv(PATH + \"train.csv\").with_columns(columns)\n",
    "test = pl.read_csv(PATH + \"test.csv\").with_columns(columns)\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "with open('/kaggle/input/english-word-hx/words.txt', 'r') as file:\n",
    "    english_vocab = set(word.strip().lower() for word in file)\n",
    "\n",
    "# Display the first sample data in the training set\n",
    "train.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ad9dfe1",
   "metadata": {
    "papermill": {
     "duration": 0.01434,
     "end_time": "2024-06-13T21:49:12.881858",
     "exception": false,
     "start_time": "2024-06-13T21:49:12.867518",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Features engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "97bee24e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:49:12.912318Z",
     "iopub.status.busy": "2024-06-13T21:49:12.912002Z",
     "iopub.status.idle": "2024-06-13T21:49:12.920087Z",
     "shell.execute_reply": "2024-06-13T21:49:12.919266Z"
    },
    "papermill": {
     "duration": 0.025334,
     "end_time": "2024-06-13T21:49:12.921897",
     "exception": false,
     "start_time": "2024-06-13T21:49:12.896563",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def count_spelling_errors(text):\n",
    "    \n",
    "    doc = nlp(text)\n",
    "    lemmatized_tokens = [token.lemma_.lower() for token in doc]\n",
    "    spelling_errors = sum(1 for token in lemmatized_tokens if token not in english_vocab)\n",
    "    return spelling_errors\n",
    "\n",
    "def removeHTML(x):\n",
    "    \n",
    "    html=re.compile(r'<.*?>')\n",
    "    return html.sub(r'',x)\n",
    "\n",
    "def dataPreprocessing(x):\n",
    "    \n",
    "    # Convert words to lowercase\n",
    "    x = x.lower()\n",
    "    # Remove HTML\n",
    "    x = removeHTML(x)\n",
    "    # Delete strings starting with @\n",
    "    x = re.sub(\"@\\w+\", '',x)\n",
    "    # Delete Numbers\n",
    "    x = re.sub(\"'\\d+\", '',x)\n",
    "    x = re.sub(\"\\d+\", '',x)\n",
    "    # Delete URL\n",
    "    x = re.sub(\"http\\w+\", '',x)\n",
    "    # Replace consecutive empty spaces with a single space character\n",
    "    x = re.sub(r\"\\s+\", \" \", x)\n",
    "    # Replace consecutive commas and periods with one comma and period character\n",
    "    x = re.sub(r\"\\.+\", \".\", x)\n",
    "    x = re.sub(r\"\\,+\", \",\", x)\n",
    "    # Remove empty characters at the beginning and end\n",
    "    x = x.strip()\n",
    "    return x"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af101f19",
   "metadata": {
    "papermill": {
     "duration": 0.014322,
     "end_time": "2024-06-13T21:49:12.950661",
     "exception": false,
     "start_time": "2024-06-13T21:49:12.936339",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Paragraph Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c23814b4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T21:49:12.981141Z",
     "iopub.status.busy": "2024-06-13T21:49:12.980870Z",
     "iopub.status.idle": "2024-06-13T22:58:49.399036Z",
     "shell.execute_reply": "2024-06-13T22:58:49.398056Z"
    },
    "papermill": {
     "duration": 4176.450922,
     "end_time": "2024-06-13T22:58:49.416110",
     "exception": false,
     "start_time": "2024-06-13T21:49:12.965188",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features Number:  56\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay_id</th>\n",
       "      <th>paragraph_&gt;0_cnt</th>\n",
       "      <th>paragraph_&gt;50_cnt</th>\n",
       "      <th>paragraph_&gt;75_cnt</th>\n",
       "      <th>paragraph_&gt;100_cnt</th>\n",
       "      <th>paragraph_&gt;125_cnt</th>\n",
       "      <th>paragraph_&gt;150_cnt</th>\n",
       "      <th>paragraph_&gt;175_cnt</th>\n",
       "      <th>paragraph_&gt;200_cnt</th>\n",
       "      <th>paragraph_&gt;250_cnt</th>\n",
       "      <th>...</th>\n",
       "      <th>paragraph_sentence_cnt_q1</th>\n",
       "      <th>paragraph_word_cnt_q1</th>\n",
       "      <th>paragraph_error_num_q3</th>\n",
       "      <th>paragraph_len_q3</th>\n",
       "      <th>paragraph_sentence_cnt_q3</th>\n",
       "      <th>paragraph_word_cnt_q3</th>\n",
       "      <th>score</th>\n",
       "      <th>no_adjectives</th>\n",
       "      <th>no_adverbs</th>\n",
       "      <th>no_mistakes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000d118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>14.0</td>\n",
       "      <td>491.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>2640.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>491.0</td>\n",
       "      <td>3</td>\n",
       "      <td>33</td>\n",
       "      <td>27</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000fe60</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>398.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>3</td>\n",
       "      <td>14</td>\n",
       "      <td>27</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>001ab80</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>101.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>927.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>165.0</td>\n",
       "      <td>4</td>\n",
       "      <td>40</td>\n",
       "      <td>56</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 58 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  essay_id  paragraph_>0_cnt  paragraph_>50_cnt  paragraph_>75_cnt  \\\n",
       "0  000d118                 1                  1                  1   \n",
       "1  000fe60                 5                  5                  5   \n",
       "2  001ab80                 4                  4                  4   \n",
       "\n",
       "   paragraph_>100_cnt  paragraph_>125_cnt  paragraph_>150_cnt  \\\n",
       "0                   1                   1                   1   \n",
       "1                   5                   5                   5   \n",
       "2                   4                   4                   4   \n",
       "\n",
       "   paragraph_>175_cnt  paragraph_>200_cnt  paragraph_>250_cnt  ...  \\\n",
       "0                   1                   1                   1  ...   \n",
       "1                   5                   4                   3  ...   \n",
       "2                   4                   4                   4  ...   \n",
       "\n",
       "   paragraph_sentence_cnt_q1  paragraph_word_cnt_q1  paragraph_error_num_q3  \\\n",
       "0                       14.0                  491.0                    27.0   \n",
       "1                        4.0                   46.0                     1.0   \n",
       "2                        5.0                  101.0                     2.0   \n",
       "\n",
       "   paragraph_len_q3  paragraph_sentence_cnt_q3  paragraph_word_cnt_q3  score  \\\n",
       "0            2640.0                       14.0                  491.0      3   \n",
       "1             398.0                        5.0                   77.0      3   \n",
       "2             927.0                        8.0                  165.0      4   \n",
       "\n",
       "   no_adjectives  no_adverbs  no_mistakes  \n",
       "0             33          27           55  \n",
       "1             14          27           24  \n",
       "2             40          56           11  \n",
       "\n",
       "[3 rows x 58 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# paragraph features\n",
    "def remove_punctuation(text):\n",
    "    \"\"\"\n",
    "    Remove all punctuation from the input text.\n",
    "    \n",
    "    Args:\n",
    "    - text (str): The input text.\n",
    "    \n",
    "    Returns:\n",
    "    - str: The text with punctuation removed.\n",
    "    \"\"\"\n",
    "    # string.punctuation\n",
    "    translator = str.maketrans('', '', string.punctuation)\n",
    "    return text.translate(translator)\n",
    "\n",
    "\n",
    "def grammar(text):\n",
    "    \n",
    "    adj_list = []\n",
    "    adv_list = []\n",
    "    mist_list = []\n",
    "  #  word_count = []\n",
    "\n",
    "    for essay in text['full_text']:\n",
    "\n",
    "        #word_count = 0\n",
    "        adj = 0\n",
    "        adv = 0\n",
    "\n",
    "        for sent in sent_tokenize(essay):\n",
    "            \n",
    "            wordtokens = word_tokenize(sent)\n",
    "           # word_count += len(wordtokens)\n",
    "            \n",
    "           # print(nltk.pos_tag(wordtokens))\n",
    "            adj += sum(1 for word, pos in nltk.pos_tag(wordtokens) if 'JJ' in pos)\n",
    "            adv += sum(1 for word, pos in nltk.pos_tag(wordtokens) if 'RB' in pos)\n",
    "\n",
    "        adj_list.append(adj)\n",
    "        adv_list.append(adv)\n",
    "        mist_list.append(len(tool.check(essay)))\n",
    "    \n",
    "    return text.with_columns([\n",
    "        pl.Series(name='no_adjectives', values=adj_list),\n",
    "        pl.Series(name='no_adverbs', values=adv_list),\n",
    "        pl.Series(name='no_mistakes', values=mist_list)\n",
    "    ])\n",
    "\n",
    "\n",
    "def Paragraph_Preprocess(tmp):\n",
    "    # Expand the paragraph list into several lines of data\n",
    "    \n",
    "    tmp = tmp.explode('paragraph')\n",
    "    \n",
    "    # Paragraph preprocessing\n",
    "    tmp = tmp.with_columns(pl.col('paragraph').map_elements(dataPreprocessing))\n",
    "    tmp = tmp.with_columns(pl.col('paragraph').map_elements(remove_punctuation).alias('paragraph_no_pinctuation'))\n",
    "    tmp = tmp.with_columns(pl.col('paragraph_no_pinctuation').map_elements(count_spelling_errors).alias(\"paragraph_error_num\"))\n",
    "    # Calculate the length of each paragraph\n",
    "    tmp = tmp.with_columns(pl.col('paragraph').map_elements(lambda x: len(x)).alias(\"paragraph_len\"))\n",
    "    # Calculate the number of sentences and words in each paragraph\n",
    "    tmp = tmp.with_columns(pl.col('paragraph').map_elements(lambda x: len(x.split('.'))).alias(\"paragraph_sentence_cnt\"),\n",
    "                    pl.col('paragraph').map_elements(lambda x: len(x.split(' '))).alias(\"paragraph_word_cnt\"),)\n",
    "    return tmp\n",
    "# feature_eng\n",
    "paragraph_fea = ['paragraph_len','paragraph_sentence_cnt','paragraph_word_cnt']\n",
    "paragraph_fea2 = ['paragraph_error_num'] + paragraph_fea\n",
    "\n",
    "def Paragraph_Eng(train_tmp):\n",
    "    num_list = [0, 50,75,100,125,150,175,200,250,300,350,400,500,600]\n",
    "    num_list2 = [0, 50,75,100,125,150,175,200,250,300,350,400,500,600,700]\n",
    "    aggs = [\n",
    "        # Count the number of paragraph lengths greater than and less than the i-value\n",
    "        *[pl.col('paragraph').filter(pl.col('paragraph_len') >= i).count().alias(f\"paragraph_{i}_cnt\") for i in [0, 50,75,100,125,150,175,200,250,300,350,400,500,600,700] ], \n",
    "        *[pl.col('paragraph').filter(pl.col('paragraph_len') <= i).count().alias(f\"paragraph_{i}_cnt\") for i in [25,49]], \n",
    "        # other\n",
    "        *[pl.col(fea).max().alias(f\"{fea}_max\") for fea in paragraph_fea2],\n",
    "        *[pl.col(fea).mean().alias(f\"{fea}_mean\") for fea in paragraph_fea2],\n",
    "        *[pl.col(fea).min().alias(f\"{fea}_min\") for fea in paragraph_fea2],\n",
    "        *[pl.col(fea).sum().alias(f\"{fea}_sum\") for fea in paragraph_fea2],\n",
    "        *[pl.col(fea).first().alias(f\"{fea}_first\") for fea in paragraph_fea2],\n",
    "        *[pl.col(fea).last().alias(f\"{fea}_last\") for fea in paragraph_fea2],\n",
    "        *[pl.col(fea).kurtosis().alias(f\"{fea}_kurtosis\") for fea in paragraph_fea2],\n",
    "        *[pl.col(fea).quantile(0.25).alias(f\"{fea}_q1\") for fea in paragraph_fea2],  \n",
    "        *[pl.col(fea).quantile(0.75).alias(f\"{fea}_q3\") for fea in paragraph_fea2],  \n",
    "        ]\n",
    "    \n",
    "    df = train_tmp.group_by(['essay_id'], maintain_order=True).agg(aggs).sort(\"essay_id\")\n",
    "    df = df.to_pandas()\n",
    "    return df\n",
    "\n",
    "if ENABLE_DONT_WASTE_YOUR_RUN_TIME:\n",
    "    with open(\"/kaggle/input/aes2-cache/paragraph_preprocess_tmp.pickle\", \"rb\") as f:\n",
    "        tmp = pickle.load(f)\n",
    "    with open(\"/kaggle/input/aes2-cache/paragraph_preprocess_train_feats.pickle\", \"rb\") as f:\n",
    "        train_feats = pickle.load(f)\n",
    "else:\n",
    "    tmp = Paragraph_Preprocess(train)\n",
    "    train_feats = Paragraph_Eng(tmp)\n",
    "\n",
    "feats_new = grammar(train) #incorporating new grammar features\n",
    "train_feats['score'] = feats_new['score'] \n",
    "train_feats['no_adjectives'] = feats_new['no_adjectives']\n",
    "train_feats['no_adverbs'] = feats_new['no_adverbs']\n",
    "train_feats['no_mistakes'] = feats_new['no_mistakes']\n",
    "\n",
    "\n",
    "# Obtain feature names\n",
    "feature_names = list(filter(lambda x: x not in ['essay_id','score'], train_feats.columns))\n",
    "print('Features Number: ',len(feature_names))\n",
    "train_feats.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "623e4945",
   "metadata": {
    "papermill": {
     "duration": 0.014618,
     "end_time": "2024-06-13T22:58:49.446005",
     "exception": false,
     "start_time": "2024-06-13T22:58:49.431387",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Sentence Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "21c341b1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T22:58:49.477781Z",
     "iopub.status.busy": "2024-06-13T22:58:49.477170Z",
     "iopub.status.idle": "2024-06-13T22:58:56.402648Z",
     "shell.execute_reply": "2024-06-13T22:58:56.401621Z"
    },
    "papermill": {
     "duration": 6.943462,
     "end_time": "2024-06-13T22:58:56.404657",
     "exception": false,
     "start_time": "2024-06-13T22:58:49.461195",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features Number:  84\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay_id</th>\n",
       "      <th>paragraph_&gt;0_cnt</th>\n",
       "      <th>paragraph_&gt;50_cnt</th>\n",
       "      <th>paragraph_&gt;75_cnt</th>\n",
       "      <th>paragraph_&gt;100_cnt</th>\n",
       "      <th>paragraph_&gt;125_cnt</th>\n",
       "      <th>paragraph_&gt;150_cnt</th>\n",
       "      <th>paragraph_&gt;175_cnt</th>\n",
       "      <th>paragraph_&gt;200_cnt</th>\n",
       "      <th>paragraph_&gt;250_cnt</th>\n",
       "      <th>...</th>\n",
       "      <th>sentence_len_first</th>\n",
       "      <th>sentence_word_cnt_first</th>\n",
       "      <th>sentence_len_last</th>\n",
       "      <th>sentence_word_cnt_last</th>\n",
       "      <th>sentence_len_kurtosis</th>\n",
       "      <th>sentence_word_cnt_kurtosis</th>\n",
       "      <th>sentence_len_q1</th>\n",
       "      <th>sentence_word_cnt_q1</th>\n",
       "      <th>sentence_len_q3</th>\n",
       "      <th>sentence_word_cnt_q3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000d118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>36</td>\n",
       "      <td>7</td>\n",
       "      <td>47</td>\n",
       "      <td>10</td>\n",
       "      <td>1.514267</td>\n",
       "      <td>2.111700</td>\n",
       "      <td>110.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>225.0</td>\n",
       "      <td>37.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000fe60</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>62</td>\n",
       "      <td>13</td>\n",
       "      <td>124</td>\n",
       "      <td>25</td>\n",
       "      <td>1.126323</td>\n",
       "      <td>0.642912</td>\n",
       "      <td>53.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>25.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>001ab80</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>144</td>\n",
       "      <td>27</td>\n",
       "      <td>58</td>\n",
       "      <td>10</td>\n",
       "      <td>-0.423362</td>\n",
       "      <td>0.129704</td>\n",
       "      <td>90.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>151.0</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 86 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  essay_id  paragraph_>0_cnt  paragraph_>50_cnt  paragraph_>75_cnt  \\\n",
       "0  000d118                 1                  1                  1   \n",
       "1  000fe60                 5                  5                  5   \n",
       "2  001ab80                 4                  4                  4   \n",
       "\n",
       "   paragraph_>100_cnt  paragraph_>125_cnt  paragraph_>150_cnt  \\\n",
       "0                   1                   1                   1   \n",
       "1                   5                   5                   5   \n",
       "2                   4                   4                   4   \n",
       "\n",
       "   paragraph_>175_cnt  paragraph_>200_cnt  paragraph_>250_cnt  ...  \\\n",
       "0                   1                   1                   1  ...   \n",
       "1                   5                   4                   3  ...   \n",
       "2                   4                   4                   4  ...   \n",
       "\n",
       "   sentence_len_first  sentence_word_cnt_first  sentence_len_last  \\\n",
       "0                  36                        7                 47   \n",
       "1                  62                       13                124   \n",
       "2                 144                       27                 58   \n",
       "\n",
       "   sentence_word_cnt_last  sentence_len_kurtosis  sentence_word_cnt_kurtosis  \\\n",
       "0                      10               1.514267                    2.111700   \n",
       "1                      25               1.126323                    0.642912   \n",
       "2                      10              -0.423362                    0.129704   \n",
       "\n",
       "   sentence_len_q1  sentence_word_cnt_q1  sentence_len_q3  \\\n",
       "0            110.0                  21.0            225.0   \n",
       "1             53.0                  13.0            124.0   \n",
       "2             90.0                  17.0            151.0   \n",
       "\n",
       "   sentence_word_cnt_q3  \n",
       "0                  37.0  \n",
       "1                  25.0  \n",
       "2                  29.0  \n",
       "\n",
       "[3 rows x 86 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sentence feature\n",
    "def Sentence_Preprocess(tmp):\n",
    "    \n",
    "    # Preprocess full_text and use periods to segment sentences in the text\n",
    "    tmp = tmp.with_columns(pl.col('full_text').map_elements(dataPreprocessing).str.split(by=\".\").alias(\"sentence\"))\n",
    "    tmp = tmp.explode('sentence')\n",
    "    \n",
    "    # Calculate the length of a sentence\n",
    "    tmp = tmp.with_columns(pl.col('sentence').map_elements(lambda x: len(x)).alias(\"sentence_len\"))\n",
    "    \n",
    "    # Filter out the portion of data with a sentence length greater than 15\n",
    "    tmp = tmp.filter(pl.col('sentence_len')>=15)\n",
    "    \n",
    "    # Count the number of words in each sentence\n",
    "    tmp = tmp.with_columns(pl.col('sentence').map_elements(lambda x: len(x.split(' '))).alias(\"sentence_word_cnt\"))\n",
    "    \n",
    "    return tmp\n",
    "\n",
    "# feature_eng\n",
    "sentence_fea = ['sentence_len','sentence_word_cnt']\n",
    "\n",
    "def Sentence_Eng(train_tmp):\n",
    "    \n",
    "    aggs = [\n",
    "        # Count the number of sentences with a length greater than i\n",
    "        *[pl.col('sentence').filter(pl.col('sentence_len') >= i).count().alias(f\"sentence_{i}_cnt\") for i in [0,15,50,100,150,200,250,300] ], \n",
    "        *[pl.col('sentence').filter(pl.col('sentence_len') <= i).count().alias(f\"sentence_<{i}_cnt\") for i in [15,50] ], \n",
    "        # other\n",
    "        *[pl.col(fea).max().alias(f\"{fea}_max\") for fea in sentence_fea],\n",
    "        *[pl.col(fea).mean().alias(f\"{fea}_mean\") for fea in sentence_fea],\n",
    "        *[pl.col(fea).min().alias(f\"{fea}_min\") for fea in sentence_fea],\n",
    "        *[pl.col(fea).sum().alias(f\"{fea}_sum\") for fea in sentence_fea],\n",
    "        *[pl.col(fea).first().alias(f\"{fea}_first\") for fea in sentence_fea],\n",
    "        *[pl.col(fea).last().alias(f\"{fea}_last\") for fea in sentence_fea],\n",
    "        *[pl.col(fea).kurtosis().alias(f\"{fea}_kurtosis\") for fea in sentence_fea],\n",
    "        *[pl.col(fea).quantile(0.25).alias(f\"{fea}_q1\") for fea in sentence_fea], \n",
    "        *[pl.col(fea).quantile(0.75).alias(f\"{fea}_q3\") for fea in sentence_fea], \n",
    "        ]\n",
    "    df = train_tmp.group_by(['essay_id'], maintain_order=True).agg(aggs).sort(\"essay_id\")\n",
    "    df = df.to_pandas()\n",
    "    return df\n",
    "\n",
    "tmp = Sentence_Preprocess(train)\n",
    "# Merge the newly generated feature data with the previously generated feature data\n",
    "train_feats = train_feats.merge(Sentence_Eng(tmp), on='essay_id', how='left')\n",
    "\n",
    "feature_names = list(filter(lambda x: x not in ['essay_id','score'], train_feats.columns))\n",
    "print('Features Number: ',len(feature_names))\n",
    "train_feats.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0189a410",
   "metadata": {
    "papermill": {
     "duration": 0.015187,
     "end_time": "2024-06-13T22:58:56.435898",
     "exception": false,
     "start_time": "2024-06-13T22:58:56.420711",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Word Feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c6b86acc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T22:58:56.467666Z",
     "iopub.status.busy": "2024-06-13T22:58:56.467357Z",
     "iopub.status.idle": "2024-06-13T22:59:09.332337Z",
     "shell.execute_reply": "2024-06-13T22:59:09.331323Z"
    },
    "papermill": {
     "duration": 12.883311,
     "end_time": "2024-06-13T22:59:09.334448",
     "exception": false,
     "start_time": "2024-06-13T22:58:56.451137",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features Number:  105\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay_id</th>\n",
       "      <th>paragraph_&gt;0_cnt</th>\n",
       "      <th>paragraph_&gt;50_cnt</th>\n",
       "      <th>paragraph_&gt;75_cnt</th>\n",
       "      <th>paragraph_&gt;100_cnt</th>\n",
       "      <th>paragraph_&gt;125_cnt</th>\n",
       "      <th>paragraph_&gt;150_cnt</th>\n",
       "      <th>paragraph_&gt;175_cnt</th>\n",
       "      <th>paragraph_&gt;200_cnt</th>\n",
       "      <th>paragraph_&gt;250_cnt</th>\n",
       "      <th>...</th>\n",
       "      <th>word_12_cnt</th>\n",
       "      <th>word_13_cnt</th>\n",
       "      <th>word_14_cnt</th>\n",
       "      <th>word_15_cnt</th>\n",
       "      <th>word_len_max</th>\n",
       "      <th>word_len_mean</th>\n",
       "      <th>word_len_std</th>\n",
       "      <th>word_len_q1</th>\n",
       "      <th>word_len_q2</th>\n",
       "      <th>word_len_q3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000d118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>25</td>\n",
       "      <td>4.378819</td>\n",
       "      <td>2.538495</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000fe60</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>4.012048</td>\n",
       "      <td>2.060968</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>001ab80</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>4.574545</td>\n",
       "      <td>2.604621</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 107 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  essay_id  paragraph_>0_cnt  paragraph_>50_cnt  paragraph_>75_cnt  \\\n",
       "0  000d118                 1                  1                  1   \n",
       "1  000fe60                 5                  5                  5   \n",
       "2  001ab80                 4                  4                  4   \n",
       "\n",
       "   paragraph_>100_cnt  paragraph_>125_cnt  paragraph_>150_cnt  \\\n",
       "0                   1                   1                   1   \n",
       "1                   5                   5                   5   \n",
       "2                   4                   4                   4   \n",
       "\n",
       "   paragraph_>175_cnt  paragraph_>200_cnt  paragraph_>250_cnt  ...  \\\n",
       "0                   1                   1                   1  ...   \n",
       "1                   5                   4                   3  ...   \n",
       "2                   4                   4                   4  ...   \n",
       "\n",
       "   word_12_cnt  word_13_cnt  word_14_cnt  word_15_cnt  word_len_max  \\\n",
       "0            6            6            5            2            25   \n",
       "1            0            0            0            0            11   \n",
       "2           14           10            5            2            15   \n",
       "\n",
       "   word_len_mean  word_len_std  word_len_q1  word_len_q2  word_len_q3  \n",
       "0       4.378819      2.538495          3.0          4.0          5.0  \n",
       "1       4.012048      2.060968          2.0          4.0          5.0  \n",
       "2       4.574545      2.604621          3.0          4.0          5.0  \n",
       "\n",
       "[3 rows x 107 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# word feature\n",
    "def Word_Preprocess(tmp):\n",
    "    # Preprocess full_text and use spaces to separate words from the text\n",
    "    tmp = tmp.with_columns(pl.col('full_text').map_elements(dataPreprocessing).str.split(by=\" \").alias(\"word\"))\n",
    "    tmp = tmp.explode('word')\n",
    "    # Calculate the length of each word\n",
    "    tmp = tmp.with_columns(pl.col('word').map_elements(lambda x: len(x)).alias(\"word_len\"))\n",
    "    # Delete data with a word length of 0\n",
    "    tmp = tmp.filter(pl.col('word_len')!=0)\n",
    "    \n",
    "    return tmp\n",
    "\n",
    "# feature_eng\n",
    "def Word_Eng(train_tmp):\n",
    "    aggs = [\n",
    "        # Count the number of words with a length greater than i+1\n",
    "        *[pl.col('word').filter(pl.col('word_len') >= i+1).count().alias(f\"word_{i+1}_cnt\") for i in range(15) ], \n",
    "        # other\n",
    "        pl.col('word_len').max().alias(f\"word_len_max\"),\n",
    "        pl.col('word_len').mean().alias(f\"word_len_mean\"),\n",
    "        pl.col('word_len').std().alias(f\"word_len_std\"),\n",
    "        pl.col('word_len').quantile(0.25).alias(f\"word_len_q1\"),\n",
    "        pl.col('word_len').quantile(0.50).alias(f\"word_len_q2\"),\n",
    "        pl.col('word_len').quantile(0.75).alias(f\"word_len_q3\"),\n",
    "        ]\n",
    "    df = train_tmp.group_by(['essay_id'], maintain_order=True).agg(aggs).sort(\"essay_id\")\n",
    "    df = df.to_pandas()\n",
    "    return df\n",
    "\n",
    "tmp = Word_Preprocess(train)\n",
    "# Merge the newly generated feature data with the previously generated feature data\n",
    "train_feats = train_feats.merge(Word_Eng(tmp), on='essay_id', how='left')\n",
    "\n",
    "feature_names = list(filter(lambda x: x not in ['essay_id','score'], train_feats.columns))\n",
    "print('Features Number: ',len(feature_names))\n",
    "train_feats.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e58ade11",
   "metadata": {
    "papermill": {
     "duration": 0.016045,
     "end_time": "2024-06-13T22:59:09.366610",
     "exception": false,
     "start_time": "2024-06-13T22:59:09.350565",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e5d4b78f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T22:59:09.399950Z",
     "iopub.status.busy": "2024-06-13T22:59:09.399645Z",
     "iopub.status.idle": "2024-06-13T23:02:22.746974Z",
     "shell.execute_reply": "2024-06-13T23:02:22.746115Z"
    },
    "papermill": {
     "duration": 193.383953,
     "end_time": "2024-06-13T23:02:22.766561",
     "exception": false,
     "start_time": "2024-06-13T22:59:09.382608",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features Number:  19732\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay_id</th>\n",
       "      <th>paragraph_&gt;0_cnt</th>\n",
       "      <th>paragraph_&gt;50_cnt</th>\n",
       "      <th>paragraph_&gt;75_cnt</th>\n",
       "      <th>paragraph_&gt;100_cnt</th>\n",
       "      <th>paragraph_&gt;125_cnt</th>\n",
       "      <th>paragraph_&gt;150_cnt</th>\n",
       "      <th>paragraph_&gt;175_cnt</th>\n",
       "      <th>paragraph_&gt;200_cnt</th>\n",
       "      <th>paragraph_&gt;250_cnt</th>\n",
       "      <th>...</th>\n",
       "      <th>tfid_19617</th>\n",
       "      <th>tfid_19618</th>\n",
       "      <th>tfid_19619</th>\n",
       "      <th>tfid_19620</th>\n",
       "      <th>tfid_19621</th>\n",
       "      <th>tfid_19622</th>\n",
       "      <th>tfid_19623</th>\n",
       "      <th>tfid_19624</th>\n",
       "      <th>tfid_19625</th>\n",
       "      <th>tfid_19626</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000d118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000fe60</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>001ab80</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 19734 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  essay_id  paragraph_>0_cnt  paragraph_>50_cnt  paragraph_>75_cnt  \\\n",
       "0  000d118                 1                  1                  1   \n",
       "1  000fe60                 5                  5                  5   \n",
       "2  001ab80                 4                  4                  4   \n",
       "\n",
       "   paragraph_>100_cnt  paragraph_>125_cnt  paragraph_>150_cnt  \\\n",
       "0                   1                   1                   1   \n",
       "1                   5                   5                   5   \n",
       "2                   4                   4                   4   \n",
       "\n",
       "   paragraph_>175_cnt  paragraph_>200_cnt  paragraph_>250_cnt  ...  \\\n",
       "0                   1                   1                   1  ...   \n",
       "1                   5                   4                   3  ...   \n",
       "2                   4                   4                   4  ...   \n",
       "\n",
       "   tfid_19617  tfid_19618  tfid_19619  tfid_19620  tfid_19621  tfid_19622  \\\n",
       "0         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "1         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "2         0.0         0.0         0.0         0.0         0.0         0.0   \n",
       "\n",
       "   tfid_19623  tfid_19624  tfid_19625  tfid_19626  \n",
       "0         0.0         0.0         0.0         0.0  \n",
       "1         0.0         0.0         0.0         0.0  \n",
       "2         0.0         0.0         0.0         0.0  \n",
       "\n",
       "[3 rows x 19734 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# TfidfVectorizer parameter\n",
    "vectorizer = TfidfVectorizer(\n",
    "            tokenizer=lambda x: x,\n",
    "            preprocessor=lambda x: x,\n",
    "            token_pattern=None,\n",
    "            strip_accents='unicode',\n",
    "            analyzer = 'word',\n",
    "            ngram_range=(3,6), #(3,6)\n",
    "            min_df=0.05,\n",
    "            max_df=0.95,\n",
    "            sublinear_tf=True,\n",
    ")\n",
    "# Fit all datasets into TfidfVector,this may cause leakage and overly optimistic CV scores\n",
    "train_tfid = vectorizer.fit_transform([i for i in train['full_text']])\n",
    "\n",
    "# Convert to array\n",
    "dense_matrix = train_tfid.toarray()\n",
    "\n",
    "# Convert to dataframe\n",
    "df = pd.DataFrame(dense_matrix)\n",
    "\n",
    "# rename features\n",
    "tfid_columns = [ f'tfid_{i}' for i in range(len(df.columns))]\n",
    "df.columns = tfid_columns\n",
    "df['essay_id'] = train_feats['essay_id']\n",
    "\n",
    "# Merge the newly generated feature data with the previously generated feature data\n",
    "train_feats = train_feats.merge(df, on='essay_id', how='left')\n",
    "\n",
    "feature_names = list(filter(lambda x: x not in ['essay_id','score'], train_feats.columns))\n",
    "print('Features Number: ',len(feature_names))\n",
    "train_feats.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "46dbfd7a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T23:02:22.800366Z",
     "iopub.status.busy": "2024-06-13T23:02:22.799833Z",
     "iopub.status.idle": "2024-06-13T23:03:38.968665Z",
     "shell.execute_reply": "2024-06-13T23:03:38.967789Z"
    },
    "papermill": {
     "duration": 76.20443,
     "end_time": "2024-06-13T23:03:38.987367",
     "exception": false,
     "start_time": "2024-06-13T23:02:22.782937",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features Number:  21902\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>essay_id</th>\n",
       "      <th>paragraph_&gt;0_cnt</th>\n",
       "      <th>paragraph_&gt;50_cnt</th>\n",
       "      <th>paragraph_&gt;75_cnt</th>\n",
       "      <th>paragraph_&gt;100_cnt</th>\n",
       "      <th>paragraph_&gt;125_cnt</th>\n",
       "      <th>paragraph_&gt;150_cnt</th>\n",
       "      <th>paragraph_&gt;175_cnt</th>\n",
       "      <th>paragraph_&gt;200_cnt</th>\n",
       "      <th>paragraph_&gt;250_cnt</th>\n",
       "      <th>...</th>\n",
       "      <th>tfid_cnt_2160</th>\n",
       "      <th>tfid_cnt_2161</th>\n",
       "      <th>tfid_cnt_2162</th>\n",
       "      <th>tfid_cnt_2163</th>\n",
       "      <th>tfid_cnt_2164</th>\n",
       "      <th>tfid_cnt_2165</th>\n",
       "      <th>tfid_cnt_2166</th>\n",
       "      <th>tfid_cnt_2167</th>\n",
       "      <th>tfid_cnt_2168</th>\n",
       "      <th>tfid_cnt_2169</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>000d118</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000fe60</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>001ab80</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 21904 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  essay_id  paragraph_>0_cnt  paragraph_>50_cnt  paragraph_>75_cnt  \\\n",
       "0  000d118                 1                  1                  1   \n",
       "1  000fe60                 5                  5                  5   \n",
       "2  001ab80                 4                  4                  4   \n",
       "\n",
       "   paragraph_>100_cnt  paragraph_>125_cnt  paragraph_>150_cnt  \\\n",
       "0                   1                   1                   1   \n",
       "1                   5                   5                   5   \n",
       "2                   4                   4                   4   \n",
       "\n",
       "   paragraph_>175_cnt  paragraph_>200_cnt  paragraph_>250_cnt  ...  \\\n",
       "0                   1                   1                   1  ...   \n",
       "1                   5                   4                   3  ...   \n",
       "2                   4                   4                   4  ...   \n",
       "\n",
       "   tfid_cnt_2160  tfid_cnt_2161  tfid_cnt_2162  tfid_cnt_2163  tfid_cnt_2164  \\\n",
       "0              3              0              0              0              0   \n",
       "1              2              0              0              1              1   \n",
       "2              1              0              2              0              0   \n",
       "\n",
       "   tfid_cnt_2165  tfid_cnt_2166  tfid_cnt_2167  tfid_cnt_2168  tfid_cnt_2169  \n",
       "0              0              0              0              0              0  \n",
       "1              0              0              0              0              0  \n",
       "2              0              0              0              0              0  \n",
       "\n",
       "[3 rows x 21904 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer_cnt = CountVectorizer(\n",
    "            tokenizer=lambda x: x,\n",
    "            preprocessor=lambda x: x,\n",
    "            token_pattern=None,\n",
    "            strip_accents='unicode',\n",
    "            analyzer = 'word',\n",
    "            ngram_range=(2,3), #(2,3)\n",
    "            min_df=0.10,\n",
    "            max_df=0.85,\n",
    ")\n",
    "train_tfid = vectorizer_cnt.fit_transform([i for i in train['full_text']])\n",
    "dense_matrix = train_tfid.toarray()\n",
    "df = pd.DataFrame(dense_matrix)\n",
    "tfid_columns = [ f'tfid_cnt_{i}' for i in range(len(df.columns))]\n",
    "df.columns = tfid_columns\n",
    "df['essay_id'] = train_feats['essay_id']\n",
    "train_feats = train_feats.merge(df, on='essay_id', how='left')\n",
    "\n",
    "feature_names = list(filter(lambda x: x not in ['essay_id','score'], train_feats.columns))\n",
    "print('Features Number: ',len(feature_names))\n",
    "train_feats.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e58ff2f3",
   "metadata": {
    "papermill": {
     "duration": 0.016105,
     "end_time": "2024-06-13T23:03:39.019841",
     "exception": false,
     "start_time": "2024-06-13T23:03:39.003736",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Deberta predictions to LGBM as features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fae85bee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T23:03:39.054312Z",
     "iopub.status.busy": "2024-06-13T23:03:39.053691Z",
     "iopub.status.idle": "2024-06-13T23:03:39.096220Z",
     "shell.execute_reply": "2024-06-13T23:03:39.095385Z"
    },
    "papermill": {
     "duration": 0.06159,
     "end_time": "2024-06-13T23:03:39.098081",
     "exception": false,
     "start_time": "2024-06-13T23:03:39.036491",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(17307, 6) (17307, 21904)\n",
      "Features Number:  21908\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(17307, 21910)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add Deberta predictions to LGBM as features\n",
    "deberta_oof = joblib.load('/kaggle/input/aes2-400-20240419134941/oof.pkl')\n",
    "print(deberta_oof.shape, train_feats.shape)\n",
    "\n",
    "for i in range(6):\n",
    "    train_feats[f'deberta_oof_{i}'] = deberta_oof[:, i]\n",
    "\n",
    "feature_names = list(filter(lambda x: x not in ['essay_id','score'], train_feats.columns))\n",
    "print('Features Number: ', len(feature_names))    \n",
    "\n",
    "train_feats.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "eb87efbf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T23:03:39.132642Z",
     "iopub.status.busy": "2024-06-13T23:03:39.132094Z",
     "iopub.status.idle": "2024-06-13T23:03:39.141184Z",
     "shell.execute_reply": "2024-06-13T23:03:39.140326Z"
    },
    "papermill": {
     "duration": 0.028322,
     "end_time": "2024-06-13T23:03:39.143036",
     "exception": false,
     "start_time": "2024-06-13T23:03:39.114714",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# idea from https://www.kaggle.com/code/rsakata/optimize-qwk-by-lgb/notebook#QWK-objective\n",
    "def quadratic_weighted_kappa(y_true, y_pred):\n",
    "    if isinstance(y_pred, xgb.QuantileDMatrix):\n",
    "        # XGB\n",
    "        y_true, y_pred = y_pred, y_true\n",
    "\n",
    "        y_true = (y_true.get_label() + a).round()\n",
    "        y_pred = (y_pred + a).clip(1, 6).round()\n",
    "        qwk = cohen_kappa_score(y_true, y_pred, weights=\"quadratic\")\n",
    "        return 'QWK', qwk\n",
    "\n",
    "    else:\n",
    "        # For lgb\n",
    "        y_true = y_true + a\n",
    "        y_pred = (y_pred + a).clip(1, 6).round()\n",
    "        qwk = cohen_kappa_score(y_true, y_pred, weights=\"quadratic\")\n",
    "        return 'QWK', qwk, True\n",
    "    \n",
    "def qwk_obj(y_true, y_pred):\n",
    "    labels = y_true + a\n",
    "    preds = y_pred + a\n",
    "    preds = preds.clip(1, 6)\n",
    "    f = 1/2*np.sum((preds-labels)**2)\n",
    "    g = 1/2*np.sum((preds-a)**2+b)\n",
    "    df = preds - labels\n",
    "    dg = preds - a\n",
    "    grad = (df/g - f*dg/g**2)*len(labels)\n",
    "    hess = np.ones(len(labels))\n",
    "    return grad, hess\n",
    "\n",
    "a = 2.998\n",
    "b = 1.092"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d46c2082",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T23:03:39.179000Z",
     "iopub.status.busy": "2024-06-13T23:03:39.178709Z",
     "iopub.status.idle": "2024-06-13T23:03:41.089117Z",
     "shell.execute_reply": "2024-06-13T23:03:41.088143Z"
    },
    "papermill": {
     "duration": 1.932,
     "end_time": "2024-06-13T23:03:41.091589",
     "exception": false,
     "start_time": "2024-06-13T23:03:39.159589",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Converting the 'text' column to string type and assigning to X\n",
    "X = train_feats[feature_names].astype(np.float32).values\n",
    "\n",
    "# Converting the 'score' column to integer type and assigning to y\n",
    "y_split = train_feats['score'].astype(int).values\n",
    "y = train_feats['score'].astype(np.float32).values-a"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba623ea9",
   "metadata": {
    "papermill": {
     "duration": 0.016422,
     "end_time": "2024-06-13T23:03:41.125209",
     "exception": false,
     "start_time": "2024-06-13T23:03:41.108787",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "ecd12d6c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T23:03:41.167103Z",
     "iopub.status.busy": "2024-06-13T23:03:41.166537Z",
     "iopub.status.idle": "2024-06-13T23:03:41.183820Z",
     "shell.execute_reply": "2024-06-13T23:03:41.182653Z"
    },
    "papermill": {
     "duration": 0.040989,
     "end_time": "2024-06-13T23:03:41.187983",
     "exception": false,
     "start_time": "2024-06-13T23:03:41.146994",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def feature_select_wrapper():\n",
    "    \"\"\"\n",
    "    lgm\n",
    "    :param train\n",
    "    :param test\n",
    "    :return\n",
    "    \"\"\"\n",
    "    # Part 1.\n",
    "    print('feature_select_wrapper...')\n",
    "    features = feature_names\n",
    "\n",
    "    skf = StratifiedKFold(n_splits=10 , shuffle=True, random_state=0) #n_splits=5\n",
    "    fse = pd.Series(0, index=features)\n",
    "    scaler = StandardScaler() \n",
    "    \n",
    "    for train_index, test_index in skf.split(X, y_split):\n",
    "\n",
    "        X_train_fold, X_test_fold = X[train_index], X[test_index]\n",
    "        y_train_fold, y_test_fold, y_test_fold_int = y[train_index], y[test_index], y_split[test_index]\n",
    "\n",
    "        # Normalize the features\n",
    "        X_train_fold = scaler.fit_transform(X_train_fold)\n",
    "        X_test_fold = scaler.transform(X_test_fold)\n",
    "        \n",
    "        model = lgb.LGBMRegressor(\n",
    "                    objective = qwk_obj,\n",
    "                    metrics = 'None',\n",
    "                    learning_rate = 0.05, #0.05\n",
    "                    max_depth = 5,\n",
    "                    num_leaves = 10, #10\n",
    "                    colsample_bytree=0.3,\n",
    "                    reg_alpha = 0.7,\n",
    "                    reg_lambda = 0.1,\n",
    "                    n_estimators=700,\n",
    "                    random_state=412,\n",
    "                    extra_trees=True,\n",
    "                    class_weight='balanced',\n",
    "                    verbosity = - 1)\n",
    "\n",
    "        predictor = model.fit(X_train_fold,\n",
    "                              y_train_fold,\n",
    "                              eval_names=['train', 'valid'],\n",
    "                              eval_set=[(X_train_fold, y_train_fold), (X_test_fold, y_test_fold)],\n",
    "                              eval_metric=quadratic_weighted_kappa,\n",
    "                              callbacks=callbacks)\n",
    "        models.append(predictor)\n",
    "        predictions_fold = predictor.predict(X_test_fold)\n",
    "        predictions_fold = predictions_fold + a\n",
    "        predictions_fold = predictions_fold.clip(1, 6).round()\n",
    "        predictions.append(predictions_fold)\n",
    "        f1_fold = f1_score(y_test_fold_int, predictions_fold, average='weighted')\n",
    "        f1_scores.append(f1_fold)\n",
    "\n",
    "        kappa_fold = cohen_kappa_score(y_test_fold_int, predictions_fold, weights='quadratic')\n",
    "        kappa_scores.append(kappa_fold)\n",
    "\n",
    "        print(f'F1 score across fold: {f1_fold}')\n",
    "        print(f'Cohen kappa score across fold: {kappa_fold}')\n",
    "\n",
    "        fse += pd.Series(predictor.feature_importances_, features)\n",
    "        if ENABLE_DONT_WASTE_YOUR_RUN_TIME:\n",
    "            break\n",
    "    \n",
    "    # Part 4.\n",
    "    feature_select = fse.sort_values(ascending=False).index.tolist()[:5000] #13000\n",
    "    print('done')\n",
    "    return feature_select"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "df42e458",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T23:03:41.228657Z",
     "iopub.status.busy": "2024-06-13T23:03:41.228339Z",
     "iopub.status.idle": "2024-06-13T23:03:41.247305Z",
     "shell.execute_reply": "2024-06-13T23:03:41.246606Z"
    },
    "papermill": {
     "duration": 0.038773,
     "end_time": "2024-06-13T23:03:41.249190",
     "exception": false,
     "start_time": "2024-06-13T23:03:41.210417",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "f1_scores = []\n",
    "kappa_scores = []\n",
    "models = []\n",
    "predictions = []\n",
    "callbacks = [log_evaluation(period=25), early_stopping(stopping_rounds=75,first_metric_only=True)]\n",
    "\n",
    "if ENABLE_DONT_WASTE_YOUR_RUN_TIME:\n",
    "    with open(\"/kaggle/input/aes2-cache/feature_select.pickle\", \"rb\") as f:\n",
    "        feature_select = pickle.load(f)\n",
    "else:\n",
    "    feature_select = feature_select_wrapper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1d14da48",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T23:03:41.283722Z",
     "iopub.status.busy": "2024-06-13T23:03:41.283473Z",
     "iopub.status.idle": "2024-06-13T23:03:49.515960Z",
     "shell.execute_reply": "2024-06-13T23:03:49.515075Z"
    },
    "papermill": {
     "duration": 8.252156,
     "end_time": "2024-06-13T23:03:49.518011",
     "exception": false,
     "start_time": "2024-06-13T23:03:41.265855",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features Select Number:  13000\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    X = train_feats[feature_select].astype(np.float32).values\n",
    "except KeyError:\n",
    "    with open(\"/kaggle/input/aes2-preprocessing/X.pickle\", \"rb\") as f:\n",
    "        X = pickle.load(f)\n",
    "    with open(\"/kaggle/input/aes2-preprocessing/y.pickle\", \"rb\") as f:\n",
    "        y = pickle.load(f)\n",
    "    with open(\"/kaggle/input/aes2-preprocessing/y_split.pickle\", \"rb\") as f:\n",
    "        y_split = pickle.load(f)\n",
    "print('Features Select Number: ', len(feature_select))  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "377d879b",
   "metadata": {
    "papermill": {
     "duration": 0.016751,
     "end_time": "2024-06-13T23:03:49.552340",
     "exception": false,
     "start_time": "2024-06-13T23:03:49.535589",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Model training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "06b01993",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T23:03:49.587455Z",
     "iopub.status.busy": "2024-06-13T23:03:49.587132Z",
     "iopub.status.idle": "2024-06-13T23:06:49.105010Z",
     "shell.execute_reply": "2024-06-13T23:06:49.103964Z"
    },
    "papermill": {
     "duration": 179.538165,
     "end_time": "2024-06-13T23:06:49.107164",
     "exception": false,
     "start_time": "2024-06-13T23:03:49.568999",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n",
      "1 warning generated.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Info] Using self-defined objective function\n",
      "Training until validation scores don't improve for 75 rounds\n",
      "[25]\ttrain's QWK: 0.779087\tvalid's QWK: 0.764434\n",
      "[50]\ttrain's QWK: 0.821432\tvalid's QWK: 0.814442\n",
      "[75]\ttrain's QWK: 0.831539\tvalid's QWK: 0.826173\n",
      "[100]\ttrain's QWK: 0.837542\tvalid's QWK: 0.827916\n",
      "[125]\ttrain's QWK: 0.841732\tvalid's QWK: 0.827514\n",
      "[150]\ttrain's QWK: 0.844854\tvalid's QWK: 0.829682\n",
      "[175]\ttrain's QWK: 0.847213\tvalid's QWK: 0.828685\n",
      "[200]\ttrain's QWK: 0.849417\tvalid's QWK: 0.829446\n",
      "[225]\ttrain's QWK: 0.851336\tvalid's QWK: 0.833724\n",
      "[250]\ttrain's QWK: 0.853347\tvalid's QWK: 0.837106\n",
      "[275]\ttrain's QWK: 0.85537\tvalid's QWK: 0.835591\n",
      "[300]\ttrain's QWK: 0.857125\tvalid's QWK: 0.83686\n",
      "[325]\ttrain's QWK: 0.858885\tvalid's QWK: 0.835312\n",
      "[350]\ttrain's QWK: 0.861452\tvalid's QWK: 0.834555\n",
      "[375]\ttrain's QWK: 0.863279\tvalid's QWK: 0.836257\n",
      "Early stopping, best iteration is:\n",
      "[312]\ttrain's QWK: 0.85794\tvalid's QWK: 0.838587\n",
      "Evaluated only: QWK\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/xgboost/sklearn.py:889: UserWarning: `eval_metric` in `fit` method is deprecated for better compatibility with scikit-learn, use `eval_metric` in constructor or`set_params` instead.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.10/site-packages/xgboost/sklearn.py:889: UserWarning: `callbacks` in `fit` method is deprecated for better compatibility with scikit-learn, use `callbacks` in constructor or`set_params` instead.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.10/site-packages/xgboost/core.py:160: UserWarning: [23:06:22] WARNING: /workspace/src/learner.cc:742: \n",
      "Parameters: { \"class_weight\", \"extra_trees\", \"metrics\", \"num_leaves\" } are not used.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-rmse:1.07496\tvalidation_0-QWK:0.46441\tvalidation_1-rmse:1.08116\tvalidation_1-QWK:0.42674\n",
      "[0]\tvalidation_0-rmse:1.07496\tvalidation_0-QWK:0.46441\tvalidation_1-rmse:1.08116\tvalidation_1-QWK:0.42674\n",
      "[1]\tvalidation_0-rmse:0.97176\tvalidation_0-QWK:0.47091\tvalidation_1-rmse:0.98227\tvalidation_1-QWK:0.44659\n",
      "[2]\tvalidation_0-rmse:0.87759\tvalidation_0-QWK:0.49067\tvalidation_1-rmse:0.89347\tvalidation_1-QWK:0.45515\n",
      "[3]\tvalidation_0-rmse:0.79483\tvalidation_0-QWK:0.56661\tvalidation_1-rmse:0.82042\tvalidation_1-QWK:0.55313\n",
      "[4]\tvalidation_0-rmse:0.72537\tvalidation_0-QWK:0.64134\tvalidation_1-rmse:0.75959\tvalidation_1-QWK:0.61686\n",
      "[5]\tvalidation_0-rmse:0.66764\tvalidation_0-QWK:0.69069\tvalidation_1-rmse:0.71077\tvalidation_1-QWK:0.65792\n",
      "[6]\tvalidation_0-rmse:0.62244\tvalidation_0-QWK:0.72506\tvalidation_1-rmse:0.67737\tvalidation_1-QWK:0.68282\n",
      "[7]\tvalidation_0-rmse:0.58356\tvalidation_0-QWK:0.76640\tvalidation_1-rmse:0.64918\tvalidation_1-QWK:0.72540\n",
      "[8]\tvalidation_0-rmse:0.55125\tvalidation_0-QWK:0.79781\tvalidation_1-rmse:0.62652\tvalidation_1-QWK:0.75488\n",
      "[9]\tvalidation_0-rmse:0.52455\tvalidation_0-QWK:0.81685\tvalidation_1-rmse:0.61010\tvalidation_1-QWK:0.76838\n",
      "[10]\tvalidation_0-rmse:0.50213\tvalidation_0-QWK:0.82832\tvalidation_1-rmse:0.59719\tvalidation_1-QWK:0.78241\n",
      "[11]\tvalidation_0-rmse:0.48420\tvalidation_0-QWK:0.85141\tvalidation_1-rmse:0.58612\tvalidation_1-QWK:0.79548\n",
      "[12]\tvalidation_0-rmse:0.46928\tvalidation_0-QWK:0.86119\tvalidation_1-rmse:0.57818\tvalidation_1-QWK:0.79895\n",
      "[13]\tvalidation_0-rmse:0.45722\tvalidation_0-QWK:0.86658\tvalidation_1-rmse:0.57243\tvalidation_1-QWK:0.80408\n",
      "[14]\tvalidation_0-rmse:0.44600\tvalidation_0-QWK:0.87266\tvalidation_1-rmse:0.56858\tvalidation_1-QWK:0.80971\n",
      "[15]\tvalidation_0-rmse:0.43651\tvalidation_0-QWK:0.87758\tvalidation_1-rmse:0.56507\tvalidation_1-QWK:0.81215\n",
      "[16]\tvalidation_0-rmse:0.42755\tvalidation_0-QWK:0.88198\tvalidation_1-rmse:0.56286\tvalidation_1-QWK:0.81495\n",
      "[17]\tvalidation_0-rmse:0.42025\tvalidation_0-QWK:0.88475\tvalidation_1-rmse:0.56106\tvalidation_1-QWK:0.81462\n",
      "[18]\tvalidation_0-rmse:0.41333\tvalidation_0-QWK:0.88778\tvalidation_1-rmse:0.55884\tvalidation_1-QWK:0.81786\n",
      "[19]\tvalidation_0-rmse:0.40713\tvalidation_0-QWK:0.89030\tvalidation_1-rmse:0.55804\tvalidation_1-QWK:0.81842\n",
      "[20]\tvalidation_0-rmse:0.40102\tvalidation_0-QWK:0.89336\tvalidation_1-rmse:0.55727\tvalidation_1-QWK:0.81812\n",
      "[21]\tvalidation_0-rmse:0.39599\tvalidation_0-QWK:0.89567\tvalidation_1-rmse:0.55734\tvalidation_1-QWK:0.82204\n",
      "[22]\tvalidation_0-rmse:0.39161\tvalidation_0-QWK:0.89816\tvalidation_1-rmse:0.55711\tvalidation_1-QWK:0.82313\n",
      "[23]\tvalidation_0-rmse:0.38830\tvalidation_0-QWK:0.89955\tvalidation_1-rmse:0.55666\tvalidation_1-QWK:0.82312\n",
      "[24]\tvalidation_0-rmse:0.38495\tvalidation_0-QWK:0.90156\tvalidation_1-rmse:0.55594\tvalidation_1-QWK:0.82486\n",
      "[25]\tvalidation_0-rmse:0.38117\tvalidation_0-QWK:0.90356\tvalidation_1-rmse:0.55601\tvalidation_1-QWK:0.82622\n",
      "[25]\tvalidation_0-rmse:0.38117\tvalidation_0-QWK:0.90356\tvalidation_1-rmse:0.55601\tvalidation_1-QWK:0.82622\n",
      "[26]\tvalidation_0-rmse:0.37749\tvalidation_0-QWK:0.90562\tvalidation_1-rmse:0.55673\tvalidation_1-QWK:0.82484\n",
      "[27]\tvalidation_0-rmse:0.37466\tvalidation_0-QWK:0.90748\tvalidation_1-rmse:0.55665\tvalidation_1-QWK:0.82600\n",
      "[28]\tvalidation_0-rmse:0.37188\tvalidation_0-QWK:0.90921\tvalidation_1-rmse:0.55670\tvalidation_1-QWK:0.82371\n",
      "[29]\tvalidation_0-rmse:0.36811\tvalidation_0-QWK:0.91065\tvalidation_1-rmse:0.55587\tvalidation_1-QWK:0.82534\n",
      "[30]\tvalidation_0-rmse:0.36589\tvalidation_0-QWK:0.91170\tvalidation_1-rmse:0.55646\tvalidation_1-QWK:0.82667\n",
      "[31]\tvalidation_0-rmse:0.36353\tvalidation_0-QWK:0.91288\tvalidation_1-rmse:0.55657\tvalidation_1-QWK:0.82891\n",
      "[32]\tvalidation_0-rmse:0.36212\tvalidation_0-QWK:0.91357\tvalidation_1-rmse:0.55645\tvalidation_1-QWK:0.82905\n",
      "[33]\tvalidation_0-rmse:0.35939\tvalidation_0-QWK:0.91517\tvalidation_1-rmse:0.55662\tvalidation_1-QWK:0.82769\n",
      "[34]\tvalidation_0-rmse:0.35743\tvalidation_0-QWK:0.91638\tvalidation_1-rmse:0.55631\tvalidation_1-QWK:0.82676\n",
      "[35]\tvalidation_0-rmse:0.35598\tvalidation_0-QWK:0.91685\tvalidation_1-rmse:0.55637\tvalidation_1-QWK:0.82676\n",
      "[36]\tvalidation_0-rmse:0.35346\tvalidation_0-QWK:0.91786\tvalidation_1-rmse:0.55627\tvalidation_1-QWK:0.82801\n",
      "[37]\tvalidation_0-rmse:0.35099\tvalidation_0-QWK:0.91902\tvalidation_1-rmse:0.55583\tvalidation_1-QWK:0.82801\n",
      "[38]\tvalidation_0-rmse:0.34926\tvalidation_0-QWK:0.91988\tvalidation_1-rmse:0.55604\tvalidation_1-QWK:0.82627\n",
      "[39]\tvalidation_0-rmse:0.34833\tvalidation_0-QWK:0.92030\tvalidation_1-rmse:0.55593\tvalidation_1-QWK:0.82564\n",
      "[40]\tvalidation_0-rmse:0.34657\tvalidation_0-QWK:0.92107\tvalidation_1-rmse:0.55661\tvalidation_1-QWK:0.82564\n",
      "[41]\tvalidation_0-rmse:0.34519\tvalidation_0-QWK:0.92200\tvalidation_1-rmse:0.55665\tvalidation_1-QWK:0.82635\n",
      "[42]\tvalidation_0-rmse:0.34347\tvalidation_0-QWK:0.92307\tvalidation_1-rmse:0.55605\tvalidation_1-QWK:0.82508\n",
      "[43]\tvalidation_0-rmse:0.34194\tvalidation_0-QWK:0.92431\tvalidation_1-rmse:0.55585\tvalidation_1-QWK:0.82849\n",
      "[44]\tvalidation_0-rmse:0.34057\tvalidation_0-QWK:0.92517\tvalidation_1-rmse:0.55576\tvalidation_1-QWK:0.82893\n",
      "[45]\tvalidation_0-rmse:0.33979\tvalidation_0-QWK:0.92587\tvalidation_1-rmse:0.55588\tvalidation_1-QWK:0.82990\n",
      "[46]\tvalidation_0-rmse:0.33737\tvalidation_0-QWK:0.92712\tvalidation_1-rmse:0.55612\tvalidation_1-QWK:0.83087\n",
      "[47]\tvalidation_0-rmse:0.33509\tvalidation_0-QWK:0.92855\tvalidation_1-rmse:0.55594\tvalidation_1-QWK:0.83085\n",
      "[48]\tvalidation_0-rmse:0.33408\tvalidation_0-QWK:0.92905\tvalidation_1-rmse:0.55576\tvalidation_1-QWK:0.83273\n",
      "[49]\tvalidation_0-rmse:0.33313\tvalidation_0-QWK:0.92948\tvalidation_1-rmse:0.55559\tvalidation_1-QWK:0.83319\n",
      "[50]\tvalidation_0-rmse:0.33159\tvalidation_0-QWK:0.93044\tvalidation_1-rmse:0.55546\tvalidation_1-QWK:0.83255\n",
      "[50]\tvalidation_0-rmse:0.33159\tvalidation_0-QWK:0.93044\tvalidation_1-rmse:0.55546\tvalidation_1-QWK:0.83255\n",
      "[51]\tvalidation_0-rmse:0.32992\tvalidation_0-QWK:0.93140\tvalidation_1-rmse:0.55513\tvalidation_1-QWK:0.83335\n",
      "[52]\tvalidation_0-rmse:0.32856\tvalidation_0-QWK:0.93240\tvalidation_1-rmse:0.55497\tvalidation_1-QWK:0.83335\n",
      "[53]\tvalidation_0-rmse:0.32657\tvalidation_0-QWK:0.93358\tvalidation_1-rmse:0.55524\tvalidation_1-QWK:0.83273\n",
      "[54]\tvalidation_0-rmse:0.32565\tvalidation_0-QWK:0.93423\tvalidation_1-rmse:0.55507\tvalidation_1-QWK:0.83229\n",
      "[55]\tvalidation_0-rmse:0.32380\tvalidation_0-QWK:0.93516\tvalidation_1-rmse:0.55502\tvalidation_1-QWK:0.83173\n",
      "[56]\tvalidation_0-rmse:0.32212\tvalidation_0-QWK:0.93582\tvalidation_1-rmse:0.55527\tvalidation_1-QWK:0.83219\n",
      "[57]\tvalidation_0-rmse:0.32131\tvalidation_0-QWK:0.93613\tvalidation_1-rmse:0.55536\tvalidation_1-QWK:0.83219\n",
      "[58]\tvalidation_0-rmse:0.32006\tvalidation_0-QWK:0.93711\tvalidation_1-rmse:0.55542\tvalidation_1-QWK:0.83219\n",
      "[59]\tvalidation_0-rmse:0.31898\tvalidation_0-QWK:0.93771\tvalidation_1-rmse:0.55533\tvalidation_1-QWK:0.83329\n",
      "[60]\tvalidation_0-rmse:0.31678\tvalidation_0-QWK:0.93902\tvalidation_1-rmse:0.55520\tvalidation_1-QWK:0.83317\n",
      "[61]\tvalidation_0-rmse:0.31551\tvalidation_0-QWK:0.93957\tvalidation_1-rmse:0.55543\tvalidation_1-QWK:0.83351\n",
      "[62]\tvalidation_0-rmse:0.31423\tvalidation_0-QWK:0.94019\tvalidation_1-rmse:0.55546\tvalidation_1-QWK:0.83289\n",
      "[63]\tvalidation_0-rmse:0.31338\tvalidation_0-QWK:0.94070\tvalidation_1-rmse:0.55518\tvalidation_1-QWK:0.83243\n",
      "[64]\tvalidation_0-rmse:0.31141\tvalidation_0-QWK:0.94182\tvalidation_1-rmse:0.55567\tvalidation_1-QWK:0.83181\n",
      "[65]\tvalidation_0-rmse:0.30889\tvalidation_0-QWK:0.94327\tvalidation_1-rmse:0.55576\tvalidation_1-QWK:0.83039\n",
      "[66]\tvalidation_0-rmse:0.30807\tvalidation_0-QWK:0.94381\tvalidation_1-rmse:0.55589\tvalidation_1-QWK:0.83039\n",
      "[67]\tvalidation_0-rmse:0.30646\tvalidation_0-QWK:0.94475\tvalidation_1-rmse:0.55596\tvalidation_1-QWK:0.82932\n",
      "[68]\tvalidation_0-rmse:0.30480\tvalidation_0-QWK:0.94544\tvalidation_1-rmse:0.55575\tvalidation_1-QWK:0.82870\n",
      "[69]\tvalidation_0-rmse:0.30276\tvalidation_0-QWK:0.94683\tvalidation_1-rmse:0.55582\tvalidation_1-QWK:0.82994\n",
      "[70]\tvalidation_0-rmse:0.30105\tvalidation_0-QWK:0.94793\tvalidation_1-rmse:0.55613\tvalidation_1-QWK:0.83028\n",
      "[71]\tvalidation_0-rmse:0.29950\tvalidation_0-QWK:0.94872\tvalidation_1-rmse:0.55615\tvalidation_1-QWK:0.82985\n",
      "[72]\tvalidation_0-rmse:0.29787\tvalidation_0-QWK:0.94951\tvalidation_1-rmse:0.55601\tvalidation_1-QWK:0.83030\n",
      "[73]\tvalidation_0-rmse:0.29679\tvalidation_0-QWK:0.95039\tvalidation_1-rmse:0.55607\tvalidation_1-QWK:0.82922\n",
      "[74]\tvalidation_0-rmse:0.29549\tvalidation_0-QWK:0.95113\tvalidation_1-rmse:0.55605\tvalidation_1-QWK:0.82968\n",
      "[75]\tvalidation_0-rmse:0.29421\tvalidation_0-QWK:0.95172\tvalidation_1-rmse:0.55611\tvalidation_1-QWK:0.82839\n",
      "[75]\tvalidation_0-rmse:0.29421\tvalidation_0-QWK:0.95172\tvalidation_1-rmse:0.55611\tvalidation_1-QWK:0.82839\n",
      "[76]\tvalidation_0-rmse:0.29318\tvalidation_0-QWK:0.95261\tvalidation_1-rmse:0.55618\tvalidation_1-QWK:0.82821\n",
      "[77]\tvalidation_0-rmse:0.29226\tvalidation_0-QWK:0.95303\tvalidation_1-rmse:0.55640\tvalidation_1-QWK:0.82867\n",
      "[78]\tvalidation_0-rmse:0.29114\tvalidation_0-QWK:0.95331\tvalidation_1-rmse:0.55582\tvalidation_1-QWK:0.82867\n",
      "[79]\tvalidation_0-rmse:0.28973\tvalidation_0-QWK:0.95397\tvalidation_1-rmse:0.55590\tvalidation_1-QWK:0.82867\n",
      "[80]\tvalidation_0-rmse:0.28898\tvalidation_0-QWK:0.95460\tvalidation_1-rmse:0.55589\tvalidation_1-QWK:0.82948\n",
      "[81]\tvalidation_0-rmse:0.28744\tvalidation_0-QWK:0.95520\tvalidation_1-rmse:0.55576\tvalidation_1-QWK:0.83120\n",
      "[82]\tvalidation_0-rmse:0.28685\tvalidation_0-QWK:0.95554\tvalidation_1-rmse:0.55576\tvalidation_1-QWK:0.83148\n",
      "[83]\tvalidation_0-rmse:0.28539\tvalidation_0-QWK:0.95621\tvalidation_1-rmse:0.55567\tvalidation_1-QWK:0.83148\n",
      "[84]\tvalidation_0-rmse:0.28375\tvalidation_0-QWK:0.95710\tvalidation_1-rmse:0.55585\tvalidation_1-QWK:0.83111\n",
      "[85]\tvalidation_0-rmse:0.28265\tvalidation_0-QWK:0.95768\tvalidation_1-rmse:0.55577\tvalidation_1-QWK:0.82977\n",
      "[86]\tvalidation_0-rmse:0.28184\tvalidation_0-QWK:0.95820\tvalidation_1-rmse:0.55576\tvalidation_1-QWK:0.82977\n",
      "[87]\tvalidation_0-rmse:0.28078\tvalidation_0-QWK:0.95886\tvalidation_1-rmse:0.55589\tvalidation_1-QWK:0.82894\n",
      "[88]\tvalidation_0-rmse:0.27984\tvalidation_0-QWK:0.95941\tvalidation_1-rmse:0.55577\tvalidation_1-QWK:0.82957\n",
      "[89]\tvalidation_0-rmse:0.27874\tvalidation_0-QWK:0.95986\tvalidation_1-rmse:0.55560\tvalidation_1-QWK:0.82873\n",
      "[90]\tvalidation_0-rmse:0.27790\tvalidation_0-QWK:0.96039\tvalidation_1-rmse:0.55557\tvalidation_1-QWK:0.82920\n",
      "[91]\tvalidation_0-rmse:0.27679\tvalidation_0-QWK:0.96081\tvalidation_1-rmse:0.55584\tvalidation_1-QWK:0.83084\n",
      "[92]\tvalidation_0-rmse:0.27561\tvalidation_0-QWK:0.96123\tvalidation_1-rmse:0.55559\tvalidation_1-QWK:0.83128\n",
      "[93]\tvalidation_0-rmse:0.27466\tvalidation_0-QWK:0.96173\tvalidation_1-rmse:0.55598\tvalidation_1-QWK:0.83082\n",
      "[94]\tvalidation_0-rmse:0.27363\tvalidation_0-QWK:0.96219\tvalidation_1-rmse:0.55589\tvalidation_1-QWK:0.83100\n",
      "[95]\tvalidation_0-rmse:0.27228\tvalidation_0-QWK:0.96281\tvalidation_1-rmse:0.55606\tvalidation_1-QWK:0.83038\n",
      "[96]\tvalidation_0-rmse:0.27122\tvalidation_0-QWK:0.96345\tvalidation_1-rmse:0.55610\tvalidation_1-QWK:0.83019\n",
      "[97]\tvalidation_0-rmse:0.27019\tvalidation_0-QWK:0.96393\tvalidation_1-rmse:0.55632\tvalidation_1-QWK:0.83019\n",
      "[98]\tvalidation_0-rmse:0.26887\tvalidation_0-QWK:0.96466\tvalidation_1-rmse:0.55616\tvalidation_1-QWK:0.83109\n",
      "[99]\tvalidation_0-rmse:0.26748\tvalidation_0-QWK:0.96518\tvalidation_1-rmse:0.55609\tvalidation_1-QWK:0.83109\n",
      "[100]\tvalidation_0-rmse:0.26631\tvalidation_0-QWK:0.96579\tvalidation_1-rmse:0.55625\tvalidation_1-QWK:0.83109\n",
      "[100]\tvalidation_0-rmse:0.26631\tvalidation_0-QWK:0.96579\tvalidation_1-rmse:0.55625\tvalidation_1-QWK:0.83109\n",
      "[101]\tvalidation_0-rmse:0.26547\tvalidation_0-QWK:0.96633\tvalidation_1-rmse:0.55630\tvalidation_1-QWK:0.83156\n",
      "[102]\tvalidation_0-rmse:0.26501\tvalidation_0-QWK:0.96656\tvalidation_1-rmse:0.55624\tvalidation_1-QWK:0.83156\n",
      "[103]\tvalidation_0-rmse:0.26410\tvalidation_0-QWK:0.96668\tvalidation_1-rmse:0.55598\tvalidation_1-QWK:0.83109\n",
      "[104]\tvalidation_0-rmse:0.26351\tvalidation_0-QWK:0.96700\tvalidation_1-rmse:0.55604\tvalidation_1-QWK:0.83137\n",
      "[105]\tvalidation_0-rmse:0.26247\tvalidation_0-QWK:0.96752\tvalidation_1-rmse:0.55607\tvalidation_1-QWK:0.83137\n",
      "[106]\tvalidation_0-rmse:0.26116\tvalidation_0-QWK:0.96806\tvalidation_1-rmse:0.55601\tvalidation_1-QWK:0.83091\n",
      "[107]\tvalidation_0-rmse:0.26007\tvalidation_0-QWK:0.96867\tvalidation_1-rmse:0.55590\tvalidation_1-QWK:0.83054\n",
      "[108]\tvalidation_0-rmse:0.25911\tvalidation_0-QWK:0.96893\tvalidation_1-rmse:0.55614\tvalidation_1-QWK:0.83054\n",
      "[109]\tvalidation_0-rmse:0.25795\tvalidation_0-QWK:0.96979\tvalidation_1-rmse:0.55585\tvalidation_1-QWK:0.83080\n",
      "[110]\tvalidation_0-rmse:0.25694\tvalidation_0-QWK:0.97019\tvalidation_1-rmse:0.55536\tvalidation_1-QWK:0.83161\n",
      "[111]\tvalidation_0-rmse:0.25568\tvalidation_0-QWK:0.97093\tvalidation_1-rmse:0.55524\tvalidation_1-QWK:0.83145\n",
      "[112]\tvalidation_0-rmse:0.25464\tvalidation_0-QWK:0.97129\tvalidation_1-rmse:0.55524\tvalidation_1-QWK:0.83145\n",
      "[113]\tvalidation_0-rmse:0.25363\tvalidation_0-QWK:0.97196\tvalidation_1-rmse:0.55526\tvalidation_1-QWK:0.83145\n",
      "[114]\tvalidation_0-rmse:0.25246\tvalidation_0-QWK:0.97222\tvalidation_1-rmse:0.55499\tvalidation_1-QWK:0.83145\n",
      "[115]\tvalidation_0-rmse:0.25145\tvalidation_0-QWK:0.97274\tvalidation_1-rmse:0.55486\tvalidation_1-QWK:0.83091\n",
      "[116]\tvalidation_0-rmse:0.25027\tvalidation_0-QWK:0.97320\tvalidation_1-rmse:0.55523\tvalidation_1-QWK:0.83047\n",
      "[117]\tvalidation_0-rmse:0.24925\tvalidation_0-QWK:0.97360\tvalidation_1-rmse:0.55510\tvalidation_1-QWK:0.83047\n",
      "[118]\tvalidation_0-rmse:0.24886\tvalidation_0-QWK:0.97386\tvalidation_1-rmse:0.55509\tvalidation_1-QWK:0.83047\n",
      "[119]\tvalidation_0-rmse:0.24768\tvalidation_0-QWK:0.97460\tvalidation_1-rmse:0.55485\tvalidation_1-QWK:0.82966\n",
      "[120]\tvalidation_0-rmse:0.24667\tvalidation_0-QWK:0.97500\tvalidation_1-rmse:0.55487\tvalidation_1-QWK:0.82966\n",
      "[121]\tvalidation_0-rmse:0.24575\tvalidation_0-QWK:0.97537\tvalidation_1-rmse:0.55499\tvalidation_1-QWK:0.82915\n",
      "[122]\tvalidation_0-rmse:0.24464\tvalidation_0-QWK:0.97591\tvalidation_1-rmse:0.55501\tvalidation_1-QWK:0.82959\n",
      "[123]\tvalidation_0-rmse:0.24350\tvalidation_0-QWK:0.97668\tvalidation_1-rmse:0.55512\tvalidation_1-QWK:0.82959\n",
      "[124]\tvalidation_0-rmse:0.24274\tvalidation_0-QWK:0.97714\tvalidation_1-rmse:0.55523\tvalidation_1-QWK:0.82988\n",
      "[125]\tvalidation_0-rmse:0.24151\tvalidation_0-QWK:0.97771\tvalidation_1-rmse:0.55541\tvalidation_1-QWK:0.83034\n",
      "[125]\tvalidation_0-rmse:0.24151\tvalidation_0-QWK:0.97771\tvalidation_1-rmse:0.55541\tvalidation_1-QWK:0.83034\n",
      "[126]\tvalidation_0-rmse:0.24089\tvalidation_0-QWK:0.97805\tvalidation_1-rmse:0.55549\tvalidation_1-QWK:0.83034\n",
      "[127]\tvalidation_0-rmse:0.24023\tvalidation_0-QWK:0.97822\tvalidation_1-rmse:0.55554\tvalidation_1-QWK:0.83126\n",
      "[128]\tvalidation_0-rmse:0.23916\tvalidation_0-QWK:0.97880\tvalidation_1-rmse:0.55545\tvalidation_1-QWK:0.83064\n",
      "[129]\tvalidation_0-rmse:0.23780\tvalidation_0-QWK:0.97952\tvalidation_1-rmse:0.55578\tvalidation_1-QWK:0.83064\n",
      "[130]\tvalidation_0-rmse:0.23677\tvalidation_0-QWK:0.97969\tvalidation_1-rmse:0.55590\tvalidation_1-QWK:0.83090\n",
      "[131]\tvalidation_0-rmse:0.23568\tvalidation_0-QWK:0.98010\tvalidation_1-rmse:0.55604\tvalidation_1-QWK:0.83090\n",
      "[132]\tvalidation_0-rmse:0.23479\tvalidation_0-QWK:0.98060\tvalidation_1-rmse:0.55598\tvalidation_1-QWK:0.83090\n",
      "[133]\tvalidation_0-rmse:0.23372\tvalidation_0-QWK:0.98109\tvalidation_1-rmse:0.55626\tvalidation_1-QWK:0.83025\n",
      "[134]\tvalidation_0-rmse:0.23234\tvalidation_0-QWK:0.98171\tvalidation_1-rmse:0.55579\tvalidation_1-QWK:0.83023\n",
      "[135]\tvalidation_0-rmse:0.23106\tvalidation_0-QWK:0.98205\tvalidation_1-rmse:0.55580\tvalidation_1-QWK:0.83069\n",
      "[136]\tvalidation_0-rmse:0.23026\tvalidation_0-QWK:0.98224\tvalidation_1-rmse:0.55580\tvalidation_1-QWK:0.83069\n",
      "[136]\tvalidation_0-rmse:0.23026\tvalidation_0-QWK:0.98224\tvalidation_1-rmse:0.55580\tvalidation_1-QWK:0.83069\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/xgboost/core.py:160: UserWarning: [23:06:48] WARNING: /workspace/src/common/error_msg.cc:58: Falling back to prediction using DMatrix due to mismatched devices. This might lead to higher memory usage and slower performance. XGBoost is running on: cuda:0, while the input data is on: cpu.\n",
      "Potential solutions:\n",
      "- Use a data structure that matches the device ordinal in the booster.\n",
      "- Set the device for booster before call to inplace_predict.\n",
      "\n",
      "This warning will only be shown once.\n",
      "\n",
      "  warnings.warn(smsg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "F1 score across fold: 0.6723215008036963\n",
      "Cohen kappa score across fold: 0.8341301935954841\n"
     ]
    }
   ],
   "source": [
    "LOAD = True # re-train\n",
    "# Define the number of splits for cross-validation\n",
    "# n_splits = 15\n",
    "n_splits = 20\n",
    "models = []\n",
    "\n",
    "\n",
    "import xgboost as xgb\n",
    "\n",
    "class Predictor:\n",
    "    def __init__(self, models: list):\n",
    "        self.models = models\n",
    "    def predict(self, X):\n",
    "        n_models = len(self.models)\n",
    "        predicted = None\n",
    "        n = 0.749\n",
    "        for i, model in enumerate(self.models):\n",
    "            if i == 0:\n",
    "                predicted = n*model.predict(X)   #0.76\n",
    "            else:\n",
    "                predicted += (1-n)*model.predict(X)  #0.24\n",
    "        return predicted\n",
    "\n",
    "\n",
    "if not LOAD:\n",
    "    for i in range(n_splits):\n",
    "        models.append(lgb.Booster(model_file=f'/kaggle/input/aes-15fold/fold_{i+1}.txt'))\n",
    "else:\n",
    "    # Initialize StratifiedKFold with the specified number of splits\n",
    "    skf = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=0)\n",
    "    # Lists to store scores\n",
    "    f1_scores = []\n",
    "    kappa_scores = []\n",
    "    models = []\n",
    "    predictions = []\n",
    "    callbacks = [log_evaluation(period=25), early_stopping(stopping_rounds=75,first_metric_only=True)]\n",
    "    # Loop through each fold of the cross-validation\n",
    "    i=1\n",
    "    for train_index, test_index in skf.split(X, y_split):\n",
    "        # Split the data into training and testing sets for this fold\n",
    "        print('fold',i)\n",
    "        X_train_fold, X_test_fold = X[train_index], X[test_index]\n",
    "        y_train_fold, y_test_fold, y_test_fold_int = y[train_index], y[test_index], y_split[test_index]\n",
    "\n",
    "        light = lgb.LGBMRegressor(\n",
    "                    objective = qwk_obj,\n",
    "                    metrics = 'None',\n",
    "                    learning_rate = 0.05, #0.05\n",
    "                    max_depth = 8,#10\n",
    "                    num_leaves = 10,#15\n",
    "                    colsample_bytree=0.3,\n",
    "                    reg_alpha = 0.7,#1\n",
    "                    reg_lambda = 0.1,\n",
    "                    n_estimators=700,\n",
    "                    random_state=42,\n",
    "                    extra_trees=True,\n",
    "                    class_weight='balanced',\n",
    "                    device='gpu' if CUDA_AVAILABLE else 'cpu',\n",
    "                    verbosity = - 1\n",
    "        )\n",
    "\n",
    "        # Fit the model on the training data for this fold  \n",
    "        light.fit(X_train_fold,\n",
    "                              y_train_fold,\n",
    "                              eval_names=['train', 'valid'],\n",
    "                              eval_set=[(X_train_fold, y_train_fold), (X_test_fold, y_test_fold)],\n",
    "                              eval_metric=quadratic_weighted_kappa,\n",
    "                              callbacks=callbacks\n",
    "                             )\n",
    "        \n",
    "        xgb_regressor = xgb.XGBRegressor(\n",
    "            objective = qwk_obj,\n",
    "            metrics = 'None',\n",
    "            learning_rate = 0.1, #0.1\n",
    "            max_depth = 8,#10\n",
    "            num_leaves = 10, #15\n",
    "            colsample_bytree=0.5,\n",
    "            reg_alpha = 0.1, #1\n",
    "            reg_lambda = 0.8, #0.1\n",
    "            n_estimators=1024,\n",
    "            random_state=42,\n",
    "            extra_trees=True,\n",
    "            class_weight='balanced',\n",
    "            tree_method=\"hist\",\n",
    "            device=\"gpu\" if CUDA_AVAILABLE else \"cpu\"\n",
    "        #             device='gpu',\n",
    "        #             verbosity = 1\n",
    "        )\n",
    "    \n",
    "        xgb_callbacks = [\n",
    "            xgb.callback.EvaluationMonitor(period=25),\n",
    "            xgb.callback.EarlyStopping(75, metric_name=\"QWK\", maximize=True, save_best=True)\n",
    "        ]\n",
    "        xgb_regressor.fit(\n",
    "            X_train_fold,\n",
    "            y_train_fold,\n",
    "            eval_set=[(X_train_fold, y_train_fold), (X_test_fold, y_test_fold)],\n",
    "            eval_metric=quadratic_weighted_kappa,\n",
    "            callbacks=xgb_callbacks\n",
    "        )\n",
    "        predictor = Predictor([light, xgb_regressor])\n",
    "        # predictor = lgb_regressor.fit(X_train_fold, y_train_fold)\n",
    "        #predictor = xgb_regressor.fit(X_train_fold, y_train_fold)\n",
    "        models.append(predictor)\n",
    "        # Make predictions on the test data for this fold\n",
    "        predictions_fold = predictor.predict(X_test_fold)\n",
    "        predictions_fold = predictions_fold + a\n",
    "        predictions_fold = predictions_fold.clip(1, 6).round()\n",
    "        predictions.append(predictions_fold)\n",
    "        # Calculate and store the F1 score for this fold\n",
    "        f1_fold = f1_score(y_test_fold_int, predictions_fold, average='weighted')\n",
    "        f1_scores.append(f1_fold)\n",
    "\n",
    "        # Calculate and store the Cohen's kappa score for this fold\n",
    "        kappa_fold = cohen_kappa_score(y_test_fold_int, predictions_fold, weights='quadratic')\n",
    "        kappa_scores.append(kappa_fold)\n",
    "#         predictor.booster_.save_model(f'fold_{i}.txt')\n",
    "\n",
    "        print(f'F1 score across fold: {f1_fold}')\n",
    "        print(f'Cohen kappa score across fold: {kappa_fold}')\n",
    "        i+=1\n",
    "        gc.collect()\n",
    "        if ENABLE_DONT_WASTE_YOUR_RUN_TIME:\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "702897ec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T23:06:49.170700Z",
     "iopub.status.busy": "2024-06-13T23:06:49.170374Z",
     "iopub.status.idle": "2024-06-13T23:06:49.176440Z",
     "shell.execute_reply": "2024-06-13T23:06:49.175504Z"
    },
    "papermill": {
     "duration": 0.039855,
     "end_time": "2024-06-13T23:06:49.178244",
     "exception": false,
     "start_time": "2024-06-13T23:06:49.138389",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean F1 score across 20 folds: 0.6723215008036963\n",
      "Mean Cohen kappa score across 20 folds: 0.8341301935954841\n"
     ]
    }
   ],
   "source": [
    "if not LOAD:\n",
    "    print(f'Mean F1 score across {n_splits} folds: 0.6694070084827064')\n",
    "    print(f'Mean Cohen kappa score across {n_splits} folds: 0.835342584985933')\n",
    "else:\n",
    "    # Calculate the mean scores across all folds\n",
    "    mean_f1_score = np.mean(f1_scores)\n",
    "    mean_kappa_score = np.mean(kappa_scores)\n",
    "    # Print the mean scores\n",
    "    print(f'Mean F1 score across {n_splits} folds: {mean_f1_score}')\n",
    "    print(f'Mean Cohen kappa score across {n_splits} folds: {mean_kappa_score}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1285b824",
   "metadata": {
    "papermill": {
     "duration": 0.03068,
     "end_time": "2024-06-13T23:06:49.239320",
     "exception": false,
     "start_time": "2024-06-13T23:06:49.208640",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "10d32de5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-06-13T23:06:49.301538Z",
     "iopub.status.busy": "2024-06-13T23:06:49.301207Z",
     "iopub.status.idle": "2024-06-13T23:06:49.321169Z",
     "shell.execute_reply": "2024-06-13T23:06:49.320356Z"
    },
    "papermill": {
     "duration": 0.053277,
     "end_time": "2024-06-13T23:06:49.323055",
     "exception": false,
     "start_time": "2024-06-13T23:06:49.269778",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if ENABLE_DONT_WASTE_YOUR_RUN_TIME:\n",
    "    import shutil\n",
    "\n",
    "    shutil.copyfile(\"/kaggle/input/learning-agency-lab-automated-essay-scoring-2/sample_submission.csv\", \"submission.csv\")\n",
    "else:\n",
    "    \n",
    "    tmp = Paragraph_Preprocess(test)\n",
    "    test_feats = Paragraph_Eng(tmp)\n",
    "    \n",
    "    #adding new grammar features\n",
    "    feats_new = grammar(test)\n",
    "    test_feats['no_adjectives'] = feats_new['no_adjectives']\n",
    "    test_feats['no_adverbs'] = feats_new['no_adverbs']\n",
    "    test_feats['no_mistakes'] = feats_new['no_mistakes']\n",
    "    \n",
    "    \n",
    "    # Sentence\n",
    "    tmp = Sentence_Preprocess(test)\n",
    "    test_feats = test_feats.merge(Sentence_Eng(tmp), on='essay_id', how='left')\n",
    "    # Word\n",
    "    tmp = Word_Preprocess(test)\n",
    "    test_feats = test_feats.merge(Word_Eng(tmp), on='essay_id', how='left')\n",
    "\n",
    "    # TfidfVectorizer\n",
    "    test_tfid = vectorizer.transform([i for i in test['full_text']])\n",
    "    dense_matrix = test_tfid.toarray()\n",
    "    df = pd.DataFrame(dense_matrix)\n",
    "    tfid_columns = [ f'tfid_{i}' for i in range(len(df.columns))]\n",
    "    df.columns = tfid_columns\n",
    "    df['essay_id'] = test_feats['essay_id']\n",
    "    test_feats = test_feats.merge(df, on='essay_id', how='left')\n",
    "\n",
    "    # CountVectorizer\n",
    "    test_tfid = vectorizer_cnt.transform([i for i in test['full_text']])\n",
    "    dense_matrix = test_tfid.toarray()\n",
    "    df = pd.DataFrame(dense_matrix)\n",
    "    tfid_columns = [ f'tfid_cnt_{i}' for i in range(len(df.columns))]\n",
    "    df.columns = tfid_columns\n",
    "    df['essay_id'] = test_feats['essay_id']\n",
    "    test_feats = test_feats.merge(df, on='essay_id', how='left')\n",
    "\n",
    "    \n",
    "    for i in range(6):\n",
    "        test_feats[f'deberta_oof_{i}'] = predicted_score[:, i]\n",
    "\n",
    "    # Features number\n",
    "    feature_names = list(filter(lambda x: x not in ['essay_id','score'], test_feats.columns))\n",
    "    print('Features number: ',len(feature_names))\n",
    "    test_feats.head(3)\n",
    "\n",
    "    # Submission\n",
    "\n",
    "    probabilities = []\n",
    "    for model in models:\n",
    "        proba = model.predict(test_feats[feature_select]) + a\n",
    "        probabilities.append(proba)\n",
    "\n",
    "    # Compute the average probabilities across all models\n",
    "    predictions = np.mean(probabilities, axis=0)\n",
    "    predictions = np.round(predictions.clip(1, 6))\n",
    "\n",
    "    # Print the predictions\n",
    "    print(predictions)\n",
    "\n",
    "    submission = pd.read_csv(\"/kaggle/input/learning-agency-lab-automated-essay-scoring-2/sample_submission.csv\")\n",
    "    submission['score'] = predictions\n",
    "    submission['score'] = submission['score'].astype(int)\n",
    "    submission.to_csv(\"submission.csv\", index=None)\n",
    "    display(submission.head())"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "databundleVersionId": 8059942,
     "sourceId": 71485,
     "sourceType": "competition"
    },
    {
     "datasetId": 3968241,
     "sourceId": 6909860,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4813598,
     "sourceId": 8141507,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4832208,
     "sourceId": 8166166,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4868756,
     "sourceId": 8214573,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4871040,
     "sourceId": 8217734,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4791897,
     "sourceId": 8339744,
     "sourceType": "datasetVersion"
    },
    {
     "sourceId": 174250445,
     "sourceType": "kernelVersion"
    }
   ],
   "dockerImageVersionId": 30674,
   "isGpuEnabled": true,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 4858.182107,
   "end_time": "2024-06-13T23:06:52.384608",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-06-13T21:45:54.202501",
   "version": "2.5.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0ba7715e1b0c477d99fa3004907d8c4c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "30bcf1efe4f24d6eaf5db9fd58710db6": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d4238551b39b47078601287870d1a0f1",
       "max": 3.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_82ca94e8c3494282b6b8c01c01ba679e",
       "value": 3.0
      }
     },
     "419fbd6e6b244a428f4cc661e623d448": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5db169a635d54816b9cc244f3ebed336": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5dfd24ba5a9f45f79906d17cc3e18a59": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "5e01b32bb0c04197bc083bc9c61b1908": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_419fbd6e6b244a428f4cc661e623d448",
       "placeholder": "​",
       "style": "IPY_MODEL_5dfd24ba5a9f45f79906d17cc3e18a59",
       "value": "100%"
      }
     },
     "82ca94e8c3494282b6b8c01c01ba679e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "a0ce432bcb1b4986bb62f98059867229": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_0ba7715e1b0c477d99fa3004907d8c4c",
       "placeholder": "​",
       "style": "IPY_MODEL_ee894102418149d8a26ac81f29d7166e",
       "value": " 3/3 [00:00&lt;00:00, 123.97ex/s]"
      }
     },
     "bff0561369cc4812835a1cd6171e0c0c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_5e01b32bb0c04197bc083bc9c61b1908",
        "IPY_MODEL_30bcf1efe4f24d6eaf5db9fd58710db6",
        "IPY_MODEL_a0ce432bcb1b4986bb62f98059867229"
       ],
       "layout": "IPY_MODEL_5db169a635d54816b9cc244f3ebed336"
      }
     },
     "d4238551b39b47078601287870d1a0f1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ee894102418149d8a26ac81f29d7166e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}