ATE/huggingface-futher-training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "fa633fb4",
   "metadata": {
    "id": "2736d24a-f23c-4bfc-9ed2-7b8a91399109"
   },
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "import datasets\n",
    "import pandas as pd\n",
    "import os\n",
    "\n",
    "TRAIN_FILE = './data/train_set_v2.csv'\n",
    "#EXTERNAL_FILE = './data/external-data.csv'\n",
    "TEST_FILE = './data/test_set_v2.csv'\n",
    "SPLIT_TRAIN_IN_VAL = False\n",
    "\n",
    "modelo = './ATE/expreriments/mdeberta-v3-base-huggingface-more-training'\n",
    "use_auth_token = None\n",
    "MAX_LEN = 512\n",
    "experiment_name = 'mdeberta-v3-base-huggingface-more-training-portuguese-data'\n",
    "model_dir ='./ATE/expreriments/' + experiment_name + '/'\n",
    "\n",
    "if not os.path.exists(model_dir):\n",
    "    os.makedirs(model_dir)\n",
    "\n",
    "train_df = pd.read_csv(TRAIN_FILE, sep=';')\n",
    "#external_df = pd.read_csv(EXTERNAL_FILE, sep=';')\n",
    "test_df = pd.read_csv(TEST_FILE, sep=';')\n",
    "\n",
    "ate_train_data = train_df.groupby('review').agg(list).reset_index()\n",
    "ate_test_data = test_df.groupby('review').agg(list).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "67a77b3c",
   "metadata": {
    "id": "a905eaf8-4426-4cb8-a65d-78e198ef1c23",
    "outputId": "89ef5188-31dc-4778-ed0a-8a0333da2d1b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: WANDB_PROJECT=ABSAPT22_ATE\n",
      "env: WANDB_WATCH=all\n",
      "env: WANDB_NOTEBOOK_NAME=experiment_name\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m WANDB_NOTEBOOK_NAME should be a path to a notebook file, couldn't find experiment_name.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33meduagarcia\u001b[0m (use `wandb login --relogin` to force relogin)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uso_no_colab = False\n",
    "if uso_no_colab:\n",
    "  !pip install transformers datasets seqeval\n",
    "  !pip install wandb\n",
    "\n",
    "%env WANDB_PROJECT=ABSAPT22_ATE\n",
    "%env WANDB_WATCH=all\n",
    "%env WANDB_NOTEBOOK_NAME=experiment_name    \n",
    "    \n",
    "import wandb\n",
    "wandb.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3cd7d2c8",
   "metadata": {
    "id": "e415e227-c13a-41e1-8cc3-8f849e78fbc6"
   },
   "outputs": [],
   "source": [
    "if SPLIT_TRAIN_IN_VAL:\n",
    "    from sklearn.model_selection import train_test_split\n",
    "    ate_train_data, ate_dev_data = train_test_split(ate_train_data, test_size=0.2)\n",
    "else:\n",
    "    ate_dev_data = ate_test_data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "668f4ab7",
   "metadata": {
    "id": "273adde0-5d02-404b-804a-4cb95c006dbd"
   },
   "outputs": [],
   "source": [
    "def is_span_a_subset(span, aspect_span):\n",
    "    if span[0] >= aspect_span[1]:\n",
    "        return False\n",
    "    elif span[1] < aspect_span[0]:\n",
    "        return False\n",
    "    else:\n",
    "        return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a09da048",
   "metadata": {
    "id": "021cdb2f-ecc3-47fb-917b-fdb3891e8a65",
    "outputId": "11758730-b654-4687-8ec0-e70ee8528e19"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "from nltk.tokenize.treebank import TreebankWordTokenizer   \n",
    "def convert_to_bio(df):\n",
    "    data = []\n",
    "    for i, row in df.iterrows():\n",
    "        #tokens = tokenize.word_tokenize(row['review'], language='portuguese')\n",
    "        aspects_span = [[i, j, p, 0] for i, j, p in zip(row['start_position'], row['end_position'], row['polarity'])]\n",
    "        tokens = []\n",
    "        ner_tags = []\n",
    "        span_generator = TreebankWordTokenizer().span_tokenize(row['review'])\n",
    "        for span in span_generator:\n",
    "            tokens.append(row['review'][span[0]:span[1]])\n",
    "            is_aspect = False\n",
    "            aspect_data = None\n",
    "            for aspect_span in aspects_span:\n",
    "                if is_span_a_subset(span, aspect_span):\n",
    "                    is_aspect = True\n",
    "                    aspect_data = aspect_span\n",
    "            if is_aspect:\n",
    "                label = 'ASPECT'\n",
    "                \n",
    "                #polarity_id = int(aspect_data[2])\n",
    "                #if polarity_id == 1:\n",
    "                #    label = 'POSITIVE'\n",
    "                #elif polarity_id == 0:\n",
    "                #    label = 'NEUTRAL'\n",
    "                #else:\n",
    "                #    label = 'NEGATIVE'\n",
    "                \n",
    "                if aspect_data[3] == 0:\n",
    "                    ner_tags.append('B-'+label)\n",
    "                    aspect_data[3] = aspect_data[3] + 1\n",
    "                else:\n",
    "                    ner_tags.append('I-'+label)\n",
    "            else:\n",
    "                ner_tags.append('O')\n",
    "        data.append({'id': i, 'tokens': tokens, 'ner_tags': ner_tags})\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ba5025c7",
   "metadata": {
    "id": "e728fef0-0105-4a1c-9cc4-081cc38d65c2"
   },
   "outputs": [],
   "source": [
    "from datasets import Dataset\n",
    "data = {\n",
    "    'train': Dataset.from_pandas(pd.DataFrame(convert_to_bio(ate_train_data))),\n",
    "    'validation':  Dataset.from_pandas(pd.DataFrame(convert_to_bio(ate_dev_data))),\n",
    "    'test':  Dataset.from_pandas(pd.DataFrame(convert_to_bio(ate_test_data)))\n",
    "}\n",
    "\n",
    "\n",
    "dataset = datasets.DatasetDict(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8542f01c",
   "metadata": {
    "id": "d110e046-b053-457b-ac67-ad6cee13b321"
   },
   "outputs": [],
   "source": [
    "sep = \" \"\n",
    "if not os.path.exists('./ATE/data/aspect/'):\n",
    "    os.makedirs('./ATE/data/aspect/')\n",
    "for data_type in dataset.keys():\n",
    "    with open(f'./ATE/data/aspect/{data_type}.conll', 'w') as f:\n",
    "        for tokens, tags in zip(dataset[data_type]['tokens'], dataset[data_type]['ner_tags']):\n",
    "            for token, tag in zip(tokens, tags):\n",
    "                f.write(str(token)+sep+tag+'\\n')\n",
    "            f.write('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fb879fbb",
   "metadata": {
    "id": "6360aca6-1f4b-4dff-8c3e-e75083aabaa6",
    "outputId": "76bdfb38-7126-47e2-cf56-c7b63dbf9434"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['B-ASPECT', 'I-ASPECT', 'O']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_list = list(set(tag for doc in dataset['train']['ner_tags'] for tag in doc))\n",
    "label_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d34c5bd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_list = ['B-ASPECT', 'O', 'I-ASPECT']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c0813a4b",
   "metadata": {
    "colab": {
     "referenced_widgets": [
      "425536fa452144d3bb683b143918467e",
      "6750a11e217d4463b09a0d89462f9ce1",
      "0ad199038d284a4a9f7315599d52e87e"
     ]
    },
    "id": "b5feee15-3280-4e94-988c-d3794bba969d",
    "outputId": "d07d4267-68cc-425d-e019-8dcee11619c5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 677/677 [00:00<00:00, 2007.95ex/s]\n",
      "100%|██████████| 170/170 [00:00<00:00, 2036.54ex/s]\n",
      "100%|██████████| 170/170 [00:00<00:00, 2052.65ex/s]\n"
     ]
    }
   ],
   "source": [
    "features = datasets.Features(\n",
    "    {\n",
    "        'id': datasets.Value('int32'),\n",
    "        'tokens': datasets.Sequence(datasets.Value('string')),\n",
    "        'ner_tags': datasets.Sequence(\n",
    "            datasets.features.ClassLabel(names=label_list)\n",
    "        )\n",
    "    }\n",
    ")\n",
    "\n",
    "dataset = dataset.map(features.encode_example, features=features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "a8d048f2",
   "metadata": {
    "id": "e82384f0-4636-4cd5-976a-d38964c46264"
   },
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "    \n",
    "tokenizer = AutoTokenizer.from_pretrained(modelo, use_auth_token=use_auth_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8960367f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "transformers.models.deberta_v2.tokenization_deberta_v2.DebertaV2Tokenizer"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.__class__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7fddffe2",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_inputs = tokenizer(dataset['train'][\"tokens\"], truncation=True, is_split_into_words=True, max_length=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c8bb5c40",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1,\n",
       " 260,\n",
       " 265,\n",
       " 2725,\n",
       " 1881,\n",
       " 14135,\n",
       " 260,\n",
       " 262,\n",
       " 520,\n",
       " 5551,\n",
       " 260,\n",
       " 263,\n",
       " 29287,\n",
       " 271,\n",
       " 7186,\n",
       " 376,\n",
       " 12299,\n",
       " 269,\n",
       " 260,\n",
       " 362,\n",
       " 674,\n",
       " 12688,\n",
       " 58522,\n",
       " 4473,\n",
       " 6544,\n",
       " 260,\n",
       " 262,\n",
       " 1635,\n",
       " 427,\n",
       " 266,\n",
       " 81385,\n",
       " 261,\n",
       " 140703,\n",
       " 23852,\n",
       " 264,\n",
       " 270,\n",
       " 29924,\n",
       " 264,\n",
       " 530,\n",
       " 47068,\n",
       " 261,\n",
       " 260,\n",
       " 14338,\n",
       " 260,\n",
       " 362,\n",
       " 14488,\n",
       " 270,\n",
       " 616,\n",
       " 94878,\n",
       " 260,\n",
       " 262,\n",
       " 260,\n",
       " 362,\n",
       " 260,\n",
       " 269,\n",
       " 5526,\n",
       " 270,\n",
       " 327,\n",
       " 43814,\n",
       " 261,\n",
       " 323,\n",
       " 299,\n",
       " 585,\n",
       " 43307,\n",
       " 270,\n",
       " 5471,\n",
       " 7461,\n",
       " 260,\n",
       " 266,\n",
       " 215299,\n",
       " 2713,\n",
       " 333,\n",
       " 263,\n",
       " 270,\n",
       " 8013,\n",
       " 11024,\n",
       " 295,\n",
       " 47603,\n",
       " 270,\n",
       " 60034,\n",
       " 5236,\n",
       " 260,\n",
       " 362,\n",
       " 1059,\n",
       " 751,\n",
       " 260,\n",
       " 262,\n",
       " 260,\n",
       " 263,\n",
       " 260,\n",
       " 27063,\n",
       " 260,\n",
       " 362,\n",
       " 1913,\n",
       " 3886,\n",
       " 15070,\n",
       " 338,\n",
       " 270,\n",
       " 333,\n",
       " 263,\n",
       " 261,\n",
       " 323,\n",
       " 3678,\n",
       " 787,\n",
       " 18241,\n",
       " 351,\n",
       " 47603,\n",
       " 270,\n",
       " 60034,\n",
       " 5236,\n",
       " 260,\n",
       " 3529,\n",
       " 16250,\n",
       " 1665,\n",
       " 2352,\n",
       " 338,\n",
       " 260,\n",
       " 262,\n",
       " 674,\n",
       " 260,\n",
       " 164900,\n",
       " 270,\n",
       " 260,\n",
       " 32746,\n",
       " 263,\n",
       " 725,\n",
       " 2228,\n",
       " 93197,\n",
       " 652,\n",
       " 1209,\n",
       " 21487,\n",
       " 450,\n",
       " 479,\n",
       " 260,\n",
       " 262,\n",
       " 376,\n",
       " 2096,\n",
       " 1356,\n",
       " 124643,\n",
       " 1519,\n",
       " 270,\n",
       " 9518,\n",
       " 4931,\n",
       " 652,\n",
       " 1209,\n",
       " 431,\n",
       " 479,\n",
       " 260,\n",
       " 261,\n",
       " 2]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_inputs['input_ids'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "7435c8ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': [260, 265], 'token_type_ids': [0, 0], 'attention_mask': [1, 1]}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer(dataset['train'][\"tokens\"][0][0], add_special_tokens=False, truncation=True, is_split_into_words=True, max_length=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "59d73a52",
   "metadata": {
    "id": "4e372509-0246-4e25-927f-f029cd5566e9"
   },
   "outputs": [],
   "source": [
    "#função que ajusta os labels para o tamanho dos textos após tokenização do BERT\n",
    "#necessário pois palavras podem ser subdivididas com ##\n",
    "def tokenize_and_align_labels(dataset_unaligned, label_all_tokens = False):\n",
    "    tokenized_inputs = tokenizer(dataset_unaligned[\"tokens\"], truncation=True, is_split_into_words=True, max_length=512)\n",
    "    #print(tokenized_inputs)\n",
    "    labels = []\n",
    "    for i, label in enumerate(dataset_unaligned[f\"ner_tags\"]):\n",
    "        word_ids = tokenized_inputs.word_ids(batch_index=i)\n",
    "        previous_word_idx = None\n",
    "        label_ids = []\n",
    "        for word_idx in word_ids:\n",
    "            # None é o valor para tokens especiais -> -100 para ignorar na função de custo\n",
    "            if word_idx is None: #special tokens\n",
    "                label_ids.append(-100)\n",
    "            #palavra nova\n",
    "            elif word_idx != previous_word_idx:\n",
    "                label_ids.append(label[word_idx])\n",
    "            #label_all_tokens = True -> mesma tag para todos os subtokens\n",
    "            #label_all_tokens = False -> apenas primeiro subtoken ganha tag\n",
    "            else: #subpalavra\n",
    "                label_ids.append(label[word_idx] if label_all_tokens else -100)\n",
    "            previous_word_idx = word_idx\n",
    "\n",
    "        labels.append(label_ids)\n",
    "\n",
    "    tokenized_inputs[\"labels\"] = labels\n",
    "    return tokenized_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8833519c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#função que ajusta os labels para o tamanho dos textos após tokenização do BERT\n",
    "#necessário pois palavras podem ser subdivididas com ##\n",
    "def tokenize_and_align_labels_python_tokenizer(dataset_unaligned, label_all_tokens = False):    \n",
    "    tokenized_inputs = tokenizer(dataset_unaligned[\"tokens\"], truncation=True, is_split_into_words=True, max_length=512)\n",
    "    special_tokens = [tokenized_inputs['input_ids'][0][0], tokenized_inputs['input_ids'][0][-1]]\n",
    "    \n",
    "    word_ids_batch = []\n",
    "    for tokens in dataset_unaligned[\"tokens\"]:\n",
    "        word_ids_batch.append([None])\n",
    "        tokens_len = 2\n",
    "        for word_idx, token in enumerate(tokens):\n",
    "            token_processed = tokenizer(token, add_special_tokens=False, truncation=True, is_split_into_words=True, max_length=512)\n",
    "            for i in range(len(token_processed['input_ids'])):\n",
    "                tokens_len += 1\n",
    "                if tokens_len <= 512:\n",
    "                    word_ids_batch[-1].append(word_idx)\n",
    "        word_ids_batch[-1].append(None)\n",
    "    #print(tokenized_inputs)\n",
    "    labels = []\n",
    "    for i, label in enumerate(dataset_unaligned[f\"ner_tags\"]):\n",
    "        word_ids = word_ids_batch[i]\n",
    "        previous_word_idx = None\n",
    "        label_ids = []\n",
    "        for word_idx in word_ids:\n",
    "            # None é o valor para tokens especiais -> -100 para ignorar na função de custo\n",
    "            if word_idx is None: #special tokens\n",
    "                label_ids.append(-100)\n",
    "            #palavra nova\n",
    "            elif word_idx != previous_word_idx:\n",
    "                label_ids.append(label[word_idx])\n",
    "            #label_all_tokens = True -> mesma tag para todos os subtokens\n",
    "            #label_all_tokens = False -> apenas primeiro subtoken ganha tag\n",
    "            else: #subpalavra\n",
    "                label_ids.append(label[word_idx] if label_all_tokens else -100)\n",
    "            previous_word_idx = word_idx\n",
    "\n",
    "        labels.append(label_ids)\n",
    "\n",
    "    tokenized_inputs[\"labels\"] = labels\n",
    "    return tokenized_inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4367e875",
   "metadata": {
    "colab": {
     "referenced_widgets": [
      "64fc09e89a3d4301b34fd608c8babf90",
      "b875f04cfbf6411cb57d9141fb2c1657",
      "af2e200cfd9f42399ca0fa01899c1ff1"
     ]
    },
    "id": "ea1b2046-0a66-48c3-98cb-8734aadc184e",
    "outputId": "5cfa0f0c-d24b-4ec9-d799-d60b30dfe117"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:04<00:00,  4.46s/ba]\n",
      "100%|██████████| 1/1 [00:01<00:00,  1.36s/ba]\n",
      "100%|██████████| 1/1 [00:01<00:00,  1.12s/ba]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'tokens', 'ner_tags', 'input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 677\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['id', 'tokens', 'ner_tags', 'input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 170\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['id', 'tokens', 'ner_tags', 'input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 170\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_datasets = dataset.map(tokenize_and_align_labels_python_tokenizer, batched=True)\n",
    "tokenized_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b4d76242",
   "metadata": {
    "id": "b2f0ccf3-3dbb-4d77-a732-120b595bd82f"
   },
   "outputs": [],
   "source": [
    "from datasets import load_metric\n",
    "metric = load_metric(\"seqeval\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "e7a54871",
   "metadata": {
    "id": "c9048111-a445-4459-86ee-3c56b83c6bb7"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def compute_metrics(p):\n",
    "    predictions, labels = p\n",
    "    predictions = np.argmax(predictions, axis=2)\n",
    "\n",
    "    # Remove ignored index (special tokens)\n",
    "    true_predictions = [\n",
    "        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",
    "        for prediction, label in zip(predictions, labels)\n",
    "    ]\n",
    "    true_labels = [\n",
    "        [label_list[l] for (p, l) in zip(prediction, label) if l != -100]\n",
    "        for prediction, label in zip(predictions, labels)\n",
    "    ]\n",
    "\n",
    "    results = metric.compute(predictions=true_predictions, references=true_labels)\n",
    "    return {\n",
    "        \"precision\": results[\"overall_precision\"],\n",
    "        \"recall\": results[\"overall_recall\"],\n",
    "        \"f1\": results[\"overall_f1\"],\n",
    "        \"accuracy\": results[\"overall_accuracy\"],\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "06b5aad8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"3\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c131418e",
   "metadata": {
    "id": "b6a9d1bf-9dd4-4e48-b93b-c1283e76c311",
    "outputId": "8945a6c2-d9d7-4b86-c0f5-5970eb796e5f"
   },
   "outputs": [],
   "source": [
    "from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer\n",
    "\n",
    "model = AutoModelForTokenClassification.from_pretrained(modelo, num_labels=len(label_list), use_auth_token=use_auth_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d7ee6c63",
   "metadata": {
    "id": "8325e636-af90-448c-96fc-b30d67707e6e"
   },
   "outputs": [],
   "source": [
    "per_device_batch_size = 8\n",
    "gradient_accumulation_steps = 1\n",
    "\n",
    "total_steps_epoch = len(dataset['train']) // (per_device_batch_size * gradient_accumulation_steps)\n",
    "\n",
    "learning_rate = 4e-5 \n",
    "num_train_epochs = round(50000/total_steps_epoch)  \n",
    "weight_decay = 0.01\n",
    "warmup_ratio=0.1                         #  primeiros 10% --> Artigo Souza 2019\n",
    "\n",
    "save_total_limit = 3\n",
    "logging_steps = total_steps_epoch#(num_train_epochs*total_steps_epoch) // 20 # 20x por treinamento\n",
    "eval_steps = logging_steps\n",
    "evaluation_strategy = 'steps'\n",
    "logging_strategy = 'steps'\n",
    "save_strategy = 'steps'\n",
    "save_steps = logging_steps\n",
    "load_best_model_at_end = True\n",
    "\n",
    "fp16 = False\n",
    "\n",
    "# folders\n",
    "\n",
    "folder_model = 'e' + str(num_train_epochs) + '_lr' + str(learning_rate)\n",
    "output_dir = model_dir + 'results'\n",
    "logging_dir = model_dir + 'results'\n",
    "# get best model through a metric\n",
    "metric_for_best_model = 'eval_f1'\n",
    "if metric_for_best_model == 'eval_f1':\n",
    "    greater_is_better = True\n",
    "elif metric_for_best_model == 'eval_loss':\n",
    "    greater_is_better = False  \n",
    "\n",
    "args = TrainingArguments(\n",
    "    output_dir=output_dir,\n",
    "    learning_rate=learning_rate,\n",
    "    per_device_train_batch_size=per_device_batch_size,\n",
    "    per_device_eval_batch_size=per_device_batch_size*2,\n",
    "    gradient_accumulation_steps=gradient_accumulation_steps,\n",
    "    num_train_epochs=num_train_epochs,\n",
    "    weight_decay=weight_decay,\n",
    "    warmup_ratio=warmup_ratio,\n",
    "    save_total_limit=save_total_limit,\n",
    "    logging_steps = logging_steps,\n",
    "    eval_steps = logging_steps,\n",
    "    load_best_model_at_end = load_best_model_at_end,\n",
    "    metric_for_best_model = metric_for_best_model,\n",
    "    greater_is_better = greater_is_better,\n",
    "    gradient_checkpointing = False,\n",
    "    do_train = True,\n",
    "    do_eval = True,\n",
    "    do_predict = True,\n",
    "    evaluation_strategy = evaluation_strategy,\n",
    "    logging_dir=logging_dir, \n",
    "    logging_strategy = logging_strategy,\n",
    "    save_strategy = save_strategy,\n",
    "    save_steps = save_steps,\n",
    "    fp16 = fp16,\n",
    "    push_to_hub=False,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "603ee9dd",
   "metadata": {
    "id": "c8ad20a7-a759-4376-8278-33f535e8e2e4"
   },
   "outputs": [],
   "source": [
    "from transformers import DataCollatorForTokenClassification\n",
    "\n",
    "data_collator = DataCollatorForTokenClassification(tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d9c9a747",
   "metadata": {
    "id": "64ba72c7-a0a4-48d1-b411-8b9d0b93646c"
   },
   "outputs": [],
   "source": [
    "from transformers.trainer_callback import EarlyStoppingCallback\n",
    "\n",
    "# wait early_stopping_patience x eval_steps before to stop the training in order to get a better model\n",
    "early_stopping_patience = save_total_limit\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    args,\n",
    "    train_dataset=tokenized_datasets[\"train\"],\n",
    "    eval_dataset=tokenized_datasets[\"validation\"],\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics,\n",
    "    callbacks=[EarlyStoppingCallback(early_stopping_patience=early_stopping_patience)],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "8503f728",
   "metadata": {
    "id": "a8458ef0-8f5b-4211-b202-42045e95cfef",
    "outputId": "5dc8029b-645c-49db-cf16-edd0c2323b65"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the training set  don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: tokens, ner_tags, id. If tokens, ner_tags, id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "/opt/conda/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "***** Running training *****\n",
      "  Num examples = 677\n",
      "  Num Epochs = 595\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 50575\n",
      "Automatic Weights & Biases logging enabled, to disable set os.environ[\"WANDB_DISABLED\"] = \"true\"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "wandb version 0.12.16 is available!  To upgrade, please run:\n",
       " $ pip install wandb --upgrade"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Tracking run with wandb version 0.12.14"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Run data is saved locally in <code>/workspace/repos/ASLB/wandb/run-20220506_204634-3748arsn</code>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "Syncing run <strong><a href=\"https://wandb.ai/eduagarcia/ABSAPT22_ATE/runs/3748arsn\" target=\"_blank\">./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results</a></strong> to <a href=\"https://wandb.ai/eduagarcia/ABSAPT22_ATE\" target=\"_blank\">Weights & Biases</a> (<a href=\"https://wandb.me/run\" target=\"_blank\">docs</a>)<br/>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='420' max='50575' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [  420/50575 02:25 < 4:51:08, 2.87 it/s, Epoch 4/595]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>84</td>\n",
       "      <td>0.019500</td>\n",
       "      <td>0.046801</td>\n",
       "      <td>0.799712</td>\n",
       "      <td>0.918874</td>\n",
       "      <td>0.855162</td>\n",
       "      <td>0.984352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>168</td>\n",
       "      <td>0.018100</td>\n",
       "      <td>0.048890</td>\n",
       "      <td>0.814371</td>\n",
       "      <td>0.900662</td>\n",
       "      <td>0.855346</td>\n",
       "      <td>0.984745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>252</td>\n",
       "      <td>0.017000</td>\n",
       "      <td>0.050844</td>\n",
       "      <td>0.814371</td>\n",
       "      <td>0.900662</td>\n",
       "      <td>0.855346</td>\n",
       "      <td>0.984666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>336</td>\n",
       "      <td>0.015100</td>\n",
       "      <td>0.053096</td>\n",
       "      <td>0.813253</td>\n",
       "      <td>0.894040</td>\n",
       "      <td>0.851735</td>\n",
       "      <td>0.984430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>420</td>\n",
       "      <td>0.014800</td>\n",
       "      <td>0.055378</td>\n",
       "      <td>0.813740</td>\n",
       "      <td>0.882450</td>\n",
       "      <td>0.846704</td>\n",
       "      <td>0.983958</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: tokens, ner_tags, id. If tokens, ner_tags, id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 170\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-84\n",
      "Configuration saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-84/config.json\n",
      "Model weights saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-84/pytorch_model.bin\n",
      "tokenizer config file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-84/tokenizer_config.json\n",
      "Special tokens file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-84/special_tokens_map.json\n",
      "added tokens file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-84/added_tokens.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: tokens, ner_tags, id. If tokens, ner_tags, id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 170\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-168\n",
      "Configuration saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-168/config.json\n",
      "Model weights saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-168/pytorch_model.bin\n",
      "tokenizer config file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-168/tokenizer_config.json\n",
      "Special tokens file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-168/special_tokens_map.json\n",
      "added tokens file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-168/added_tokens.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: tokens, ner_tags, id. If tokens, ner_tags, id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 170\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-252\n",
      "Configuration saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-252/config.json\n",
      "Model weights saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-252/pytorch_model.bin\n",
      "tokenizer config file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-252/tokenizer_config.json\n",
      "Special tokens file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-252/special_tokens_map.json\n",
      "added tokens file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-252/added_tokens.json\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: tokens, ner_tags, id. If tokens, ner_tags, id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 170\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-336\n",
      "Configuration saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-336/config.json\n",
      "Model weights saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-336/pytorch_model.bin\n",
      "tokenizer config file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-336/tokenizer_config.json\n",
      "Special tokens file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-336/special_tokens_map.json\n",
      "added tokens file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-336/added_tokens.json\n",
      "Deleting older checkpoint [ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-84] due to args.save_total_limit\n",
      "The following columns in the evaluation set  don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: tokens, ner_tags, id. If tokens, ner_tags, id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 170\n",
      "  Batch size = 16\n",
      "Saving model checkpoint to ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-420\n",
      "Configuration saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-420/config.json\n",
      "Model weights saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-420/pytorch_model.bin\n",
      "tokenizer config file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-420/tokenizer_config.json\n",
      "Special tokens file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-420/special_tokens_map.json\n",
      "added tokens file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-420/added_tokens.json\n",
      "Deleting older checkpoint [ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-252] due to args.save_total_limit\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "Loading best model from ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/results/checkpoint-168 (score: 0.8553459119496857).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=420, training_loss=0.016910333292824882, metrics={'train_runtime': 150.2903, 'train_samples_per_second': 2680.247, 'train_steps_per_second': 336.515, 'total_flos': 323346657299886.0, 'train_loss': 0.016910333292824882, 'epoch': 4.94})"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "1e4e8257",
   "metadata": {
    "id": "72aa8dc1-2c96-4c29-b556-42fbcd7dfc3d",
    "outputId": "39889d7c-348d-479d-fb50-5ffaad40a82d"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the evaluation set  don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: tokens, ner_tags, id. If tokens, ner_tags, id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 170\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='33' max='11' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [11/11 00:24]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.04889018088579178,\n",
       " 'eval_precision': 0.8143712574850299,\n",
       " 'eval_recall': 0.9006622516556292,\n",
       " 'eval_f1': 0.8553459119496857,\n",
       " 'eval_accuracy': 0.9847448297554454,\n",
       " 'eval_runtime': 0.7361,\n",
       " 'eval_samples_per_second': 230.961,\n",
       " 'eval_steps_per_second': 14.945,\n",
       " 'epoch': 4.94}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "018538ba",
   "metadata": {
    "id": "fd741e29-817f-4cf1-91cd-b1b59848a4dc",
    "outputId": "2a0ae5ad-b53f-4de5-b4a8-1b2f4686878d"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the test set  don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: tokens, ner_tags, id. If tokens, ner_tags, id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Prediction *****\n",
      "  Num examples = 170\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'ASPECT': {'precision': 0.8143712574850299,\n",
       "  'recall': 0.9006622516556292,\n",
       "  'f1': 0.8553459119496857,\n",
       "  'number': 604},\n",
       " 'overall_precision': 0.8143712574850299,\n",
       " 'overall_recall': 0.9006622516556292,\n",
       " 'overall_f1': 0.8553459119496857,\n",
       " 'overall_accuracy': 0.9847448297554454}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions, labels, _ = trainer.predict(tokenized_datasets[\"validation\"])\n",
    "predictions = np.argmax(predictions, axis=2)\n",
    "\n",
    "# Remove ignored index (special tokens)\n",
    "true_predictions = [\n",
    "    [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",
    "    for prediction, label in zip(predictions, labels)\n",
    "]\n",
    "true_labels = [\n",
    "    [label_list[l] for (p, l) in zip(prediction, label) if l != -100]\n",
    "    for prediction, label in zip(predictions, labels)\n",
    "]\n",
    "\n",
    "results = metric.compute(predictions=true_predictions, references=true_labels)\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5c422b7d",
   "metadata": {
    "id": "52ae96db-a44c-49e9-9d63-d5dc1d2eaa28",
    "outputId": "cbdfdf37-e60c-49da-ec48-ae818c451c83"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Saving model checkpoint to ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/\n",
      "Configuration saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/config.json\n",
      "Model weights saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/pytorch_model.bin\n",
      "tokenizer config file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/tokenizer_config.json\n",
      "Special tokens file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/special_tokens_map.json\n",
      "added tokens file saved in ./ATE/expreriments/mdeberta-v3-base-huggingface-more-training-portuguese-data/added_tokens.json\n"
     ]
    }
   ],
   "source": [
    "trainer.save_model(model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "a21794da",
   "metadata": {
    "id": "a456f359-6eec-4155-bd87-c8b5fd50f09a",
    "outputId": "d9d1d1e8-80ee-4f30-ca26-f28096df4472"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ASPECT</th>\n",
       "      <th>overall_precision</th>\n",
       "      <th>overall_recall</th>\n",
       "      <th>overall_f1</th>\n",
       "      <th>overall_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.855346</td>\n",
       "      <td>0.814371</td>\n",
       "      <td>0.900662</td>\n",
       "      <td>0.855346</td>\n",
       "      <td>0.984745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>number</th>\n",
       "      <td>604.000000</td>\n",
       "      <td>0.814371</td>\n",
       "      <td>0.900662</td>\n",
       "      <td>0.855346</td>\n",
       "      <td>0.984745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.814371</td>\n",
       "      <td>0.814371</td>\n",
       "      <td>0.900662</td>\n",
       "      <td>0.855346</td>\n",
       "      <td>0.984745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.900662</td>\n",
       "      <td>0.814371</td>\n",
       "      <td>0.900662</td>\n",
       "      <td>0.855346</td>\n",
       "      <td>0.984745</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               ASPECT  overall_precision  overall_recall  overall_f1  \\\n",
       "f1           0.855346           0.814371        0.900662    0.855346   \n",
       "number     604.000000           0.814371        0.900662    0.855346   \n",
       "precision    0.814371           0.814371        0.900662    0.855346   \n",
       "recall       0.900662           0.814371        0.900662    0.855346   \n",
       "\n",
       "           overall_accuracy  \n",
       "f1                 0.984745  \n",
       "number             0.984745  \n",
       "precision          0.984745  \n",
       "recall             0.984745  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "resultado_eval = pd.DataFrame(results)\n",
    "resultado_eval.to_csv(model_dir+'metricas_validation.csv')\n",
    "resultado_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "2e263458",
   "metadata": {
    "id": "e5be9bfe-526d-466e-8a27-9a8131c097b3",
    "outputId": "aa837d0d-4ef6-4d68-f569-808b2aa4bd4e"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The following columns in the test set  don't have a corresponding argument in `DebertaV2ForTokenClassification.forward` and have been ignored: tokens, ner_tags, id. If tokens, ner_tags, id are not expected by `DebertaV2ForTokenClassification.forward`,  you can safely ignore this message.\n",
      "***** Running Prediction *****\n",
      "  Num examples = 170\n",
      "  Batch size = 16\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'ASPECT': {'precision': 0.8143712574850299,\n",
       "  'recall': 0.9006622516556292,\n",
       "  'f1': 0.8553459119496857,\n",
       "  'number': 604},\n",
       " 'overall_precision': 0.8143712574850299,\n",
       " 'overall_recall': 0.9006622516556292,\n",
       " 'overall_f1': 0.8553459119496857,\n",
       " 'overall_accuracy': 0.9847448297554454}"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_predictions, labels, _ = trainer.predict(tokenized_datasets[\"test\"])\n",
    "predictions = np.argmax(raw_predictions, axis=2)\n",
    "\n",
    "# Remove ignored index (special tokens)\n",
    "true_predictions = [\n",
    "    [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",
    "    for prediction, label in zip(predictions, labels)\n",
    "]\n",
    "true_labels = [\n",
    "    [label_list[l] for (p, l) in zip(prediction, label) if l != -100]\n",
    "    for prediction, label in zip(predictions, labels)\n",
    "]\n",
    "\n",
    "results = metric.compute(predictions=true_predictions, references=true_labels)\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "2bd4bbaa",
   "metadata": {
    "id": "415f2358-d5e9-4f77-8acb-9710cb8d6359",
    "outputId": "8b7d4cab-1b9d-48dc-a325-3dd7a19d79eb"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ASPECT</th>\n",
       "      <th>overall_precision</th>\n",
       "      <th>overall_recall</th>\n",
       "      <th>overall_f1</th>\n",
       "      <th>overall_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>f1</th>\n",
       "      <td>0.855346</td>\n",
       "      <td>0.814371</td>\n",
       "      <td>0.900662</td>\n",
       "      <td>0.855346</td>\n",
       "      <td>0.984745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>number</th>\n",
       "      <td>604.000000</td>\n",
       "      <td>0.814371</td>\n",
       "      <td>0.900662</td>\n",
       "      <td>0.855346</td>\n",
       "      <td>0.984745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>precision</th>\n",
       "      <td>0.814371</td>\n",
       "      <td>0.814371</td>\n",
       "      <td>0.900662</td>\n",
       "      <td>0.855346</td>\n",
       "      <td>0.984745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.900662</td>\n",
       "      <td>0.814371</td>\n",
       "      <td>0.900662</td>\n",
       "      <td>0.855346</td>\n",
       "      <td>0.984745</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               ASPECT  overall_precision  overall_recall  overall_f1  \\\n",
       "f1           0.855346           0.814371        0.900662    0.855346   \n",
       "number     604.000000           0.814371        0.900662    0.855346   \n",
       "precision    0.814371           0.814371        0.900662    0.855346   \n",
       "recall       0.900662           0.814371        0.900662    0.855346   \n",
       "\n",
       "           overall_accuracy  \n",
       "f1                 0.984745  \n",
       "number             0.984745  \n",
       "precision          0.984745  \n",
       "recall             0.984745  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resultado_teste = pd.DataFrame(results)\n",
    "resultado_teste.to_csv(model_dir+'metricas_test.csv')\n",
    "resultado_teste"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "42130f90",
   "metadata": {
    "id": "e5cf8e0b-b84f-406c-950b-63e34615ebed"
   },
   "outputs": [],
   "source": [
    "o_index = label_list.index('O')\n",
    "preds_index = np.asarray([i for i in range(len(label_list)) if i != o_index])\n",
    "\n",
    "y_pred = []\n",
    "y_true = []\n",
    "\n",
    "for sentence_raw_prediction, sentence_labels in zip(raw_predictions, true_labels):\n",
    "    for raw_prediction, true_label in zip(sentence_raw_prediction, sentence_labels):\n",
    "        if true_label.startswith('B'):\n",
    "            best_pred_idx = np.argmax(raw_prediction[preds_index])\n",
    "            best_pred = preds_index[best_pred_idx]\n",
    "            pred_label = label_list[best_pred][2:]\n",
    "            true_label = true_label[2:]\n",
    "            y_pred.append(pred_label)\n",
    "            y_true.append(true_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "479fb2d5",
   "metadata": {
    "id": "e3374b9d-c307-4c2e-bc47-283eae23bdb4",
    "outputId": "551f70a0-8a66-4e7f-8e82-6f275ce4072f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ASPECT       1.00      1.00      1.00       604\n",
      "\n",
      "    accuracy                           1.00       604\n",
      "   macro avg       1.00      1.00      1.00       604\n",
      "weighted avg       1.00      1.00      1.00       604\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_true, y_pred))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "allenlp-ner.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}