RuPERTa_base_finetuned_POS.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "RuPERTa-base-finetuned-POS.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyPzPXUUNbI0HBHEvq5u8Aog",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mrm8488/shared_colab_notebooks/blob/master/RuPERTa_base_finetuned_POS.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jwjZQY-0HRnZ",
        "colab_type": "text"
      },
      "source": [
        "# How to fine tune RuPERTa-base for POS downstream task\n",
        "\n",
        "> Creator: [Manuel Romero](https://twitter.com/mrm8488)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1RZoIhDp6_gZ",
        "colab_type": "code",
        "outputId": "1011d84e-d46f-4cd1-de97-9ca214046949",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        }
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Thu May 14 21:15:13 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 440.82       Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   39C    P0    27W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V6x-N0B2_uaf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!git clone https://github.com/huggingface/transformers"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lCyYhYhKaOWE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q ./transformers\n",
        "!pip install -q tensorboardX\n",
        "!pip install -q seqeval"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S21uyqecFGAW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget https://raw.githubusercontent.com/mrm8488/POS_es_processed/master/train.txt\n",
        "!wget https://raw.githubusercontent.com/mrm8488/POS_es_processed/master/dev.txt\n",
        "!wget https://raw.githubusercontent.com/mrm8488/POS_es_processed/master/labels.txt"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ADQzWVIiFPz4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir pos_dataset"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l4aiDhKjwxxa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wget \"https://raw.githubusercontent.com/stefan-it/fine-tuned-berts-seq/master/scripts/preprocess.py\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xITTGIAYZ_f8",
        "colab_type": "code",
        "outputId": "adb22578-3beb-4870-fa8d-5a8a2d893222",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "!python3 preprocess.py train.txt \"mrm8488/RuPERTa-base\" 128 > /content/pos_dataset/train.txt\n",
        "!python3 preprocess.py dev.txt \"mrm8488/RuPERTa-base\" 128 > /content/pos_dataset/dev.txt"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2020-05-14 22:27:12.952453: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n",
            "2020-05-14 22:27:31.869175: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gcbt5h-ZqlQ5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install -q wandb"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lQfsM0tUqz9V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!wandb login"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WV04I6DUABbA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Start tensorboard.\n",
        "%load_ext tensorboard\n",
        "%tensorboard --logdir /content/runs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QMM7MdIkrMz0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%env WANDB_PROJECT=\"fine-tune-RuPERTa-spanish-POS\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_NqPOJbZBQip",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!python /content/transformers/examples/token-classification/run_ner.py --data_dir /content/pos_dataset \\\n",
        "--model_type roberta \\\n",
        "--labels /content/labels.txt \\\n",
        "--model_name_or_path mrm8488/RuPERTa-base \\\n",
        "--output_dir model_output \\\n",
        "--max_seq_length 128 \\\n",
        "--num_train_epochs 10 \\\n",
        "--per_gpu_train_batch_size 64 \\\n",
        "--per_gpu_eval_batch_size 32 \\\n",
        "--overwrite_output_dir \\\n",
        "--save_steps 1000 \\\n",
        "--do_train \\\n",
        "--do_eval"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KWQR4YXVIrpC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mkdir /content/RuPERTa-base-finetuned-pos"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UVexIAN_b-V1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!mv /content/model_output/* /content/RuPERTa-base-finetuned-pos"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oa5PeicK1M6M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!rm -rf /content/RuPERTa-base-finetuned-pos/checkpoint-*"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "42gTofjD3Ulj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!transformers-cli login"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HNBprf1l3doo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!transformers-cli upload ./RuPERTa-base-finetuned-pos"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IROrEyjxIvbq",
        "colab_type": "text"
      },
      "source": [
        "## Test the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Cgem6ORZX6V1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from transformers import pipeline"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LgOiw-HlYELb",
        "colab_type": "code",
        "outputId": "f8305822-7603-486d-9841-5ed8a398d92a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "nlp_ner = pipeline(\n",
        "    \"ner\",\n",
        "    model=\"mrm8488/RuPERTa-base-finetuned-pos\",\n",
        "    tokenizer=\"mrm8488/RuPERTa-base-finetuned-pos\")\n",
        "\n",
        "text = 'Mis amigos están pensando viajar a Londres este verano.'\n",
        "\n",
        "nlp_ner(text)[1:-1]"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'entity': 'NUM', 'score': 0.9980951547622681, 'word': 'ĠMis'},\n",
              " {'entity': 'PRON', 'score': 0.5426339507102966, 'word': 'Ġamigos'},\n",
              " {'entity': 'AUX', 'score': 0.9093925952911377, 'word': 'ĠestÃ¡n'},\n",
              " {'entity': 'ADV', 'score': 0.6312834620475769, 'word': 'Ġpensando'},\n",
              " {'entity': 'VERB', 'score': 0.3273673355579376, 'word': 'Ġviajar'},\n",
              " {'entity': 'ADP', 'score': 0.9884780049324036, 'word': 'Ġa'},\n",
              " {'entity': 'NOUN', 'score': 0.4820754826068878, 'word': 'ĠLondres'},\n",
              " {'entity': 'DET', 'score': 0.820732593536377, 'word': 'Ġeste'},\n",
              " {'entity': 'NOUN', 'score': 0.8927345275878906, 'word': 'Ġverano'},\n",
              " {'entity': 'PUNCT', 'score': 0.999255359172821, 'word': '.'}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_odDDcx7fNWL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}