finetune_model.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BILnkSfx_vIE"
      },
      "source": [
        "## Remember to enable GPU on colab by going to `Runtime -> Change runtime type`"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b9imCxEw4ao9"
      },
      "source": [
        "Generate a Github Personal access token by following the instructions [here](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "rej6rU3N2upI",
        "outputId": "5f9383fc-9eb6-49af-998b-480c0c68d8d5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Cloning into 'ling-573-group-repo'...\n",
            "remote: Enumerating objects: 588, done.\u001b[K\n",
            "remote: Counting objects: 100% (118/118), done.\u001b[K\n",
            "remote: Compressing objects: 100% (85/85), done.\u001b[K\n",
            "remote: Total 588 (delta 49), reused 74 (delta 26), pack-reused 470\u001b[K\n",
            "Receiving objects: 100% (588/588), 105.50 MiB | 38.33 MiB/s, done.\n",
            "Resolving deltas: 100% (324/324), done.\n",
            "Collecting datasets\n",
            "  Downloading datasets-2.2.2-py3-none-any.whl (346 kB)\n",
            "\u001b[K     |████████████████████████████████| 346 kB 29.8 MB/s \n",
            "\u001b[?25hCollecting fsspec[http]>=2021.05.0\n",
            "  Downloading fsspec-2022.5.0-py3-none-any.whl (140 kB)\n",
            "\u001b[K     |████████████████████████████████| 140 kB 64.9 MB/s \n",
            "\u001b[?25hCollecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: dill<0.3.5 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.11.3)\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.8.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (1.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1 MB 59.0 MB/s \n",
            "\u001b[?25hCollecting xxhash\n",
            "  Downloading xxhash-3.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[K     |████████████████████████████████| 212 kB 71.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.64.0)\n",
            "Collecting huggingface-hub<1.0.0,>=0.1.0\n",
            "  Downloading huggingface_hub-0.6.0-py3-none-any.whl (84 kB)\n",
            "\u001b[K     |████████████████████████████████| 84 kB 4.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.21.6)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (4.2.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.13)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.7.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.9)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2021.10.8)\n",
            "Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1\n",
            "  Downloading urllib3-1.25.11-py2.py3-none-any.whl (127 kB)\n",
            "\u001b[K     |████████████████████████████████| 127 kB 66.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.4.0)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (144 kB)\n",
            "\u001b[K     |████████████████████████████████| 144 kB 49.7 MB/s \n",
            "\u001b[?25hCollecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.2.0-py3-none-any.whl (8.2 kB)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.12)\n",
            "Collecting asynctest==0.13.0\n",
            "  Downloading asynctest-0.13.0-py3-none-any.whl (26 kB)\n",
            "Collecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.7.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (271 kB)\n",
            "\u001b[K     |████████████████████████████████| 271 kB 77.5 MB/s \n",
            "\u001b[?25hCollecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (94 kB)\n",
            "\u001b[K     |████████████████████████████████| 94 kB 4.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.8.0)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2022.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: multidict, frozenlist, yarl, urllib3, asynctest, async-timeout, aiosignal, fsspec, aiohttp, xxhash, responses, huggingface-hub, datasets\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 1.24.3\n",
            "    Uninstalling urllib3-1.24.3:\n",
            "      Successfully uninstalled urllib3-1.24.3\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "Successfully installed aiohttp-3.8.1 aiosignal-1.2.0 async-timeout-4.0.2 asynctest-0.13.0 datasets-2.2.2 frozenlist-1.3.0 fsspec-2022.5.0 huggingface-hub-0.6.0 multidict-6.0.2 responses-0.18.0 urllib3-1.25.11 xxhash-3.0.0 yarl-1.7.2\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.19.2-py3-none-any.whl (4.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 4.2 MB 29.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.0)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (596 kB)\n",
            "\u001b[K     |████████████████████████████████| 596 kB 59.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.11.3)\n",
            "Collecting tokenizers!=0.11.3,<0.13,>=0.11.1\n",
            "  Downloading tokenizers-0.12.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (6.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 50.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.2.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.8.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.25.11)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Installing collected packages: pyyaml, tokenizers, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed pyyaml-6.0 tokenizers-0.12.1 transformers-4.19.2\n",
            "Collecting emoji\n",
            "  Downloading emoji-1.7.0.tar.gz (175 kB)\n",
            "\u001b[K     |████████████████████████████████| 175 kB 29.1 MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: emoji\n",
            "  Building wheel for emoji (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for emoji: filename=emoji-1.7.0-py3-none-any.whl size=171046 sha256=575cf20efd5023d03ee34754543716073bca80be3ccb36511b6773fcbbd501f7\n",
            "  Stored in directory: /root/.cache/pip/wheels/8a/4e/b6/57b01db010d17ef6ea9b40300af725ef3e210cb1acfb7ac8b6\n",
            "Successfully built emoji\n",
            "Installing collected packages: emoji\n",
            "Successfully installed emoji-1.7.0\n",
            "Collecting spacymoji\n",
            "  Downloading spacymoji-3.0.1-py2.py3-none-any.whl (8.5 kB)\n",
            "Collecting spacy<4.0.0,>=3.0.0\n",
            "  Downloading spacy-3.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (6.2 MB)\n",
            "\u001b[K     |████████████████████████████████| 6.2 MB 40.1 MB/s \n",
            "\u001b[?25hCollecting emoji<1.0.0,>=0.4.5\n",
            "  Downloading emoji-0.6.0.tar.gz (51 kB)\n",
            "\u001b[K     |████████████████████████████████| 51 kB 8.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<4.0.0,>=3.0.0->spacymoji) (3.0.6)\n",
            "Collecting typer<0.5.0,>=0.3.0\n",
            "  Downloading typer-0.4.1-py3-none-any.whl (27 kB)\n",
            "Collecting spacy-loggers<2.0.0,>=1.0.0\n",
            "  Downloading spacy_loggers-1.0.2-py3-none-any.whl (7.2 kB)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<4.0.0,>=3.0.0->spacymoji) (21.3)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<4.0.0,>=3.0.0->spacymoji) (0.4.1)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<4.0.0,>=3.0.0->spacymoji) (1.0.7)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<4.0.0,>=3.0.0->spacymoji) (2.23.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<4.0.0,>=3.0.0->spacymoji) (2.0.6)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<4.0.0,>=3.0.0->spacymoji) (57.4.0)\n",
            "Collecting pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4\n",
            "  Downloading pydantic-1.8.2-cp37-cp37m-manylinux2014_x86_64.whl (10.1 MB)\n",
            "\u001b[K     |████████████████████████████████| 10.1 MB 60.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<4.0.0,>=3.0.0->spacymoji) (1.21.6)\n",
            "Collecting typing-extensions<4.0.0.0,>=3.7.4\n",
            "  Downloading typing_extensions-3.10.0.2-py3-none-any.whl (26 kB)\n",
            "Collecting pathy>=0.3.5\n",
            "  Downloading pathy-0.6.1-py3-none-any.whl (42 kB)\n",
            "\u001b[K     |████████████████████████████████| 42 kB 1.9 MB/s \n",
            "\u001b[?25hCollecting thinc<8.1.0,>=8.0.14\n",
            "  Downloading thinc-8.0.16-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (660 kB)\n",
            "\u001b[K     |████████████████████████████████| 660 kB 66.0 MB/s \n",
            "\u001b[?25hCollecting srsly<3.0.0,>=2.4.3\n",
            "  Downloading srsly-2.4.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (457 kB)\n",
            "\u001b[K     |████████████████████████████████| 457 kB 57.4 MB/s \n",
            "\u001b[?25hCollecting langcodes<4.0.0,>=3.2.0\n",
            "  Downloading langcodes-3.3.0-py3-none-any.whl (181 kB)\n",
            "\u001b[K     |████████████████████████████████| 181 kB 77.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.7/dist-packages (from spacy<4.0.0,>=3.0.0->spacymoji) (0.9.1)\n",
            "Collecting catalogue<2.1.0,>=2.0.6\n",
            "  Downloading catalogue-2.0.7-py3-none-any.whl (17 kB)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<4.0.0,>=3.0.0->spacymoji) (4.64.0)\n",
            "Collecting spacy-legacy<3.1.0,>=3.0.9\n",
            "  Downloading spacy_legacy-3.0.9-py2.py3-none-any.whl (20 kB)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<4.0.0,>=3.0.0->spacymoji) (2.11.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy<4.0.0,>=3.0.0->spacymoji) (3.8.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<4.0.0,>=3.0.0->spacymoji) (3.0.9)\n",
            "Collecting smart-open<6.0.0,>=5.0.0\n",
            "  Downloading smart_open-5.2.1-py3-none-any.whl (58 kB)\n",
            "\u001b[K     |████████████████████████████████| 58 kB 7.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<4.0.0,>=3.0.0->spacymoji) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<4.0.0,>=3.0.0->spacymoji) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<4.0.0,>=3.0.0->spacymoji) (1.25.11)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<4.0.0,>=3.0.0->spacymoji) (3.0.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy<4.0.0,>=3.0.0->spacymoji) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<4.0.0,>=3.0.0->spacymoji) (2.0.1)\n",
            "Building wheels for collected packages: emoji\n",
            "  Building wheel for emoji (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for emoji: filename=emoji-0.6.0-py3-none-any.whl size=49734 sha256=9270d64d0c1f1f2fc92627c21d8f92882913b88fa153b4fffc5d7f47ec7515ac\n",
            "  Stored in directory: /root/.cache/pip/wheels/4e/bf/6b/2e22b3708d14bf6384f862db539b044d6931bd6b14ad3c9adc\n",
            "Successfully built emoji\n",
            "Installing collected packages: typing-extensions, catalogue, typer, srsly, smart-open, pydantic, thinc, spacy-loggers, spacy-legacy, pathy, langcodes, spacy, emoji, spacymoji\n",
            "  Attempting uninstall: typing-extensions\n",
            "    Found existing installation: typing-extensions 4.2.0\n",
            "    Uninstalling typing-extensions-4.2.0:\n",
            "      Successfully uninstalled typing-extensions-4.2.0\n",
            "  Attempting uninstall: catalogue\n",
            "    Found existing installation: catalogue 1.0.0\n",
            "    Uninstalling catalogue-1.0.0:\n",
            "      Successfully uninstalled catalogue-1.0.0\n",
            "  Attempting uninstall: srsly\n",
            "    Found existing installation: srsly 1.0.5\n",
            "    Uninstalling srsly-1.0.5:\n",
            "      Successfully uninstalled srsly-1.0.5\n",
            "  Attempting uninstall: smart-open\n",
            "    Found existing installation: smart-open 6.0.0\n",
            "    Uninstalling smart-open-6.0.0:\n",
            "      Successfully uninstalled smart-open-6.0.0\n",
            "  Attempting uninstall: thinc\n",
            "    Found existing installation: thinc 7.4.0\n",
            "    Uninstalling thinc-7.4.0:\n",
            "      Successfully uninstalled thinc-7.4.0\n",
            "  Attempting uninstall: spacy\n",
            "    Found existing installation: spacy 2.2.4\n",
            "    Uninstalling spacy-2.2.4:\n",
            "      Successfully uninstalled spacy-2.2.4\n",
            "  Attempting uninstall: emoji\n",
            "    Found existing installation: emoji 1.7.0\n",
            "    Uninstalling emoji-1.7.0:\n",
            "      Successfully uninstalled emoji-1.7.0\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "tensorflow 2.8.0+zzzcolab20220506162203 requires tf-estimator-nightly==2.8.0.dev2021122109, which is not installed.\u001b[0m\n",
            "Successfully installed catalogue-2.0.7 emoji-0.6.0 langcodes-3.3.0 pathy-0.6.1 pydantic-1.8.2 smart-open-5.2.1 spacy-3.3.0 spacy-legacy-3.0.9 spacy-loggers-1.0.2 spacymoji-3.0.1 srsly-2.4.3 thinc-8.0.16 typer-0.4.1 typing-extensions-3.10.0.2\n",
            "\u001b[38;5;3m⚠ As of spaCy v3.0, shortcuts like 'en' are deprecated. Please use the\n",
            "full pipeline package name 'en_core_web_sm' instead.\u001b[0m\n",
            "Collecting en-core-web-sm==3.3.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.3.0/en_core_web_sm-3.3.0-py3-none-any.whl (12.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 12.8 MB 25.4 MB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.4.0,>=3.3.0.dev0 in /usr/local/lib/python3.7/dist-packages (from en-core-web-sm==3.3.0) (3.3.0)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.9)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.4.3)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.23.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (4.64.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (57.4.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.8.2)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.0.2)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.21.6)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.14 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (8.0.16)\n",
            "Requirement already satisfied: typing-extensions<4.0.0.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.10.0.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.11.3)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.9.1)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.0.7)\n",
            "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.4.1)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.3.0)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.6.1)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.6)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.7)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (21.3)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.6)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (0.4.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.8.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.9)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.0.0 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (5.2.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (1.25.11)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (3.0.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.4.0,>=3.3.0.dev0->en-core-web-sm==3.3.0) (2.0.1)\n",
            "Installing collected packages: en-core-web-sm\n",
            "  Attempting uninstall: en-core-web-sm\n",
            "    Found existing installation: en-core-web-sm 2.2.5\n",
            "    Uninstalling en-core-web-sm-2.2.5:\n",
            "      Successfully uninstalled en-core-web-sm-2.2.5\n",
            "Successfully installed en-core-web-sm-3.3.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_sm')\n"
          ]
        },
        {
          "data": {
            "text/plain": []
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "%%shell \n",
        "\n",
        "# Replace <YOUR_ACCESS_TOKEN> with your generated access token\n",
        "access_token=<YOUR_ACCESS_TOKEN>\n",
        "git clone https://${access_token}@github.com/kvah/ling-573-group-repo.git\n",
        "\n",
        "pip install datasets\n",
        "pip install transformers\n",
        "pip install emoji\n",
        "pip install spacymoji\n",
        "pip install unidecode\n",
        "python -m spacy download en"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Bf4QrXhUyvlo"
      },
      "source": [
        "## Training Script"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qt6BTgs87NEy",
        "outputId": "410c2f33-04d5-41a3-80ea-d59b6f708808"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/ling-573-group-repo\n"
          ]
        }
      ],
      "source": [
        "%cd ling-573-group-repo/"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "collapsed": true,
        "id": "UUOMfIiAzg_M",
        "outputId": "6000a52c-9e5f-48f9-f839-75d56a592c39"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at xlm-roberta-large were not used when initializing XLMRobertaForSequenceClassification: ['lm_head.layer_norm.weight', 'lm_head.dense.weight', 'roberta.pooler.dense.bias', 'lm_head.decoder.weight', 'lm_head.bias', 'roberta.pooler.dense.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.bias']\n",
            "- This IS expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing XLMRobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of XLMRobertaForSequenceClassification were not initialized from the model checkpoint at xlm-roberta-large and are newly initialized: ['classifier.out_proj.weight', 'classifier.dense.bias', 'classifier.out_proj.bias', 'classifier.dense.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "100% 12/12 [00:02<00:00,  4.01ba/s]\n",
            "100% 2/2 [00:00<00:00,  7.62ba/s]\n",
            " 20% 5/25 [00:03<00:15,  1.30it/s]Epoch 0 train loss: 1.0958645343780518\n",
            "Epoch 0 Macro F1 Score: {'f1': 0.39845524761472056}\n",
            "Saving best model at epoch: 0\n",
            " 40% 10/25 [02:16<02:34, 10.31s/it]Epoch 1 train loss: 1.0398196578025818\n",
            "Epoch 1 Macro F1 Score: {'f1': 0.39845524761472056}\n",
            " 60% 15/25 [04:36<02:01, 12.19s/it]Epoch 2 train loss: 0.8706792891025543\n",
            "Epoch 2 Macro F1 Score: {'f1': 0.39845524761472056}\n",
            " 80% 20/25 [06:59<01:03, 12.72s/it]Epoch 3 train loss: 0.7799816926320394\n",
            "Epoch 3 Macro F1 Score: {'f1': 0.39845524761472056}\n",
            "100% 25/25 [09:23<00:00, 12.89s/it]Epoch 4 train loss: 0.7250549892584482\n",
            "Epoch 4 Macro F1 Score: {'f1': 0.39845524761472056}\n",
            "Time Elapsed (Finetuning): 730.6836626529694\n",
            "100% 25/25 [11:53<00:00, 28.55s/it]\n"
          ]
        },
        {
          "data": {
            "text/plain": []
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "%%shell\n",
        "\n",
        "config_name='finetune_xlmr_large'\n",
        "\n",
        "# Test whether condor is using GPU\n",
        "python3 src/test_gpu.py\n",
        "\n",
        "# Preprocess tweets\n",
        "python3 src/preprocess_olid.py \\\n",
        "    --file data/olid-training-v1.0.tsv \\\n",
        "    --train_ids data/eng_train_ids.txt \\\n",
        "    --val_ids data/eng_val_ids.txt \\\n",
        "    --split_punctuation \\\n",
        "    --remove_apostraphes \\\n",
        "    --remove_hashtags\n",
        "\n",
        "# Finetune pretrained model on training data\n",
        "python3 src/finetune_pretrained.py \\\n",
        "    --train_data data/clean_train_english.tsv \\\n",
        "    --val_data data/clean_val_english.tsv \\\n",
        "    --config configs/${config_name}.json\n",
        "\n",
        "# Run finetuned model predictions and generate output\n",
        "python3 src/finetune_predict.py \\\n",
        "    --val_data data/clean_val_english.tsv \\\n",
        "    --config configs/${config_name}.json \\\n",
        "    --model_path models/${config_name} \\\n",
        "    --val_output_csv outputs/D4/D4_english_preds.csv\n",
        "\n",
        "# Evaluation script\n",
        "python3 src/eval.py \\\n",
        "    --val_output_csv outputs/D4/D4_english_preds.csv \\\n",
        "    --output_path results/D4_english_scores.out\n",
        "\n",
        "# Sync model to drive for storage\n",
        "rsync -r /content/ling-573-group-repo/models /content/drive/MyDrive/"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "finetune_model.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}