pretrain_03_brt_pret.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pretrain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "PRETRAIN = \"brt\"\n",
    "SLUG = \"bs2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bert-base-uncased'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_name = dict(\n",
    "    brt=\"bert-base-uncased\",\n",
    "    rbt=\"roberta-base\"\n",
    ")[PRETRAIN]\n",
    "model_name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from forgebox.imports import *\n",
    "from transformers import (AutoModel,AutoModelForMaskedLM, \n",
    "                          AutoTokenizer, LineByLineTextDataset,\n",
    "                          DataCollatorForLanguageModeling,\n",
    "                          Trainer, TrainingArguments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat(list(map(pd.read_csv,[\"train.csv\", \"test.csv\"])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a dataset based on purely text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['excerpt'] = data['excerpt'].apply(lambda x: x.replace('\\n',''))\n",
    "\n",
    "text  = '\\n'.join(data.excerpt.tolist())\n",
    "\n",
    "with open('text.txt','w') as f:\n",
    "    f.write(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "model = AutoModelForMaskedLM.from_pretrained(model_name)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)\n",
    "tokenizer.save_pretrained(f'./{PRETRAIN}_{SLUG}_pre');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Pretrain use Masked Language Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.7/site-packages/transformers/data/datasets/language_modeling.py:124: FutureWarning: This dataset will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/master/examples/pytorch/language-modeling/run_mlm.py\n",
      "  FutureWarning,\n"
     ]
    }
   ],
   "source": [
    "train_dataset = LineByLineTextDataset(\n",
    "    tokenizer=tokenizer,\n",
    "    file_path=\"text.txt\", #mention train text file here\n",
    "    block_size=256)\n",
    "\n",
    "valid_dataset = LineByLineTextDataset(\n",
    "    tokenizer=tokenizer,\n",
    "    file_path=\"text.txt\", #mention valid text file here\n",
    "    block_size=256)\n",
    "\n",
    "data_collator = DataCollatorForLanguageModeling(\n",
    "    tokenizer=tokenizer, mlm=True, mlm_probability=0.15)\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=f\"./weights/{PRETRAIN}_rbt_{SLUG}\", #select model path for checkpoint\n",
    "    overwrite_output_dir=True,\n",
    "    num_train_epochs=5,\n",
    "    per_device_train_batch_size=12,\n",
    "    per_device_eval_batch_size=18,\n",
    "    evaluation_strategy= 'steps',\n",
    "    save_total_limit=2,\n",
    "    eval_steps=250,\n",
    "    metric_for_best_model='eval_loss',\n",
    "    greater_is_better=False,\n",
    "    load_best_model_at_end =True,\n",
    "    prediction_loss_only=True,\n",
    "    report_to = \"none\")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    data_collator=data_collator,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=valid_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples = 2841\n",
      "  Num Epochs = 5\n",
      "  Instantaneous batch size per device = 12\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 24\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 595\n",
      "/anaconda3/lib/python3.7/site-packages/torch/nn/parallel/_functions.py:64: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='595' max='595' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [595/595 04:21, Epoch 5/5]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>No log</td>\n",
       "      <td>1.738972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>1.877700</td>\n",
       "      <td>1.664761</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 2841\n",
      "  Batch size = 36\n",
      "Saving model checkpoint to ./weights/brt_rbt_bs2/checkpoint-250\n",
      "Configuration saved in ./weights/brt_rbt_bs2/checkpoint-250/config.json\n",
      "Model weights saved in ./weights/brt_rbt_bs2/checkpoint-250/pytorch_model.bin\n",
      "/anaconda3/lib/python3.7/site-packages/torch/nn/parallel/_functions.py:64: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 2841\n",
      "  Batch size = 36\n",
      "Saving model checkpoint to ./weights/brt_rbt_bs2/checkpoint-500\n",
      "Configuration saved in ./weights/brt_rbt_bs2/checkpoint-500/config.json\n",
      "Model weights saved in ./weights/brt_rbt_bs2/checkpoint-500/pytorch_model.bin\n",
      "/anaconda3/lib/python3.7/site-packages/torch/nn/parallel/_functions.py:64: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./weights/brt_rbt_bs2/checkpoint-500 (score: 1.6647613048553467).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=595, training_loss=1.8641962644432772, metrics={'train_runtime': 264.9274, 'train_samples_per_second': 53.618, 'train_steps_per_second': 2.246, 'total_flos': 1861958654433900.0, 'train_loss': 1.8641962644432772, 'epoch': 5.0})"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to weights/pre_brtbs2\n",
      "Configuration saved in weights/pre_brtbs2/config.json\n",
      "Model weights saved in weights/pre_brtbs2/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "trainer.save_model(f\"weights/pre_{PRETRAIN}{SLUG}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}