examples/02 - CoNLL 2003.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "! mkdir -p conll03\n",
    "! wget -q -O conll03/train.conll https://raw.githubusercontent.com/davidsbatista/NER-datasets/master/CONLL2003/train.txt\n",
    "! wget -q -O conll03/valid.conll https://raw.githubusercontent.com/davidsbatista/NER-datasets/master/CONLL2003/valid.txt\n",
    "! wget -q -O conll03/test.conll https://raw.githubusercontent.com/davidsbatista/NER-datasets/master/CONLL2003/test.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-DOCSTART- -X- -X- O\r\n",
      "\r\n",
      "EU NNP B-NP B-ORG\r\n",
      "rejects VBZ B-VP O\r\n",
      "German JJ B-NP B-MISC\r\n",
      "call NN I-NP O\r\n",
      "to TO B-VP O\r\n",
      "boycott VB I-VP O\r\n",
      "British JJ B-NP B-MISC\r\n",
      "lamb NN I-NP O\r\n",
      ". . O O\r\n",
      "\r\n",
      "Peter NNP B-NP B-PER\r\n",
      "Blackburn NNP I-NP I-PER\r\n",
      "\r\n",
      "BRUSSELS NNP B-NP B-LOC\r\n",
      "1996-08-22 CD I-NP O\r\n",
      "\r\n",
      "The DT B-NP O\r\n",
      "European NNP I-NP B-ORG\r\n"
     ]
    }
   ],
   "source": [
    "! head -n 20 conll03/train.conll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<spacy_crfsuite.crf_extractor.CRFExtractor at 0x10d393e80>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from spacy_crfsuite import CRFExtractor\n",
    "\n",
    "component_config = {\n",
    "    \"features\": [\n",
    "        [\n",
    "            \"low\",\n",
    "            \"title\",\n",
    "            \"upper\",\n",
    "            \"pos\",\n",
    "            \"pos2\"\n",
    "        ],\n",
    "        [\n",
    "            \"low\",\n",
    "            \"bias\",\n",
    "            \"prefix5\",\n",
    "            \"prefix2\",\n",
    "            \"suffix5\",\n",
    "            \"suffix3\",\n",
    "            \"suffix2\",\n",
    "            \"upper\",\n",
    "            \"title\",\n",
    "            \"digit\",\n",
    "            \"pos\",\n",
    "            \"pos2\"\n",
    "        ],\n",
    "        [\n",
    "            \"low\",\n",
    "            \"title\",\n",
    "            \"upper\",\n",
    "            \"pos\",\n",
    "            \"pos2\"\n",
    "        ],\n",
    "    ],\n",
    "    \"c1\": 0.01,\n",
    "    \"c2\": 0.22\n",
    "}\n",
    "\n",
    "crf_extractor = CRFExtractor(component_config=component_config)\n",
    "crf_extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/talmago/git/spacy_crfsuite/.venv/lib/python3.6/site-packages/spacy/util.py:275: UserWarning: [W031] Model 'en_core_web_sm' (2.2.5) requires spaCy v2.2 and is incompatible with the current spaCy version (2.3.2). This may lead to unexpected results or runtime errors. To resolve this, download a newer compatible model or retrain your custom model with the current spaCy version. For more details and available updates, run: python -m spacy validate\n",
      "  warnings.warn(warn_msg)\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "use_dense_features = crf_extractor.use_dense_features()\n",
    "\n",
    "if use_dense_features:\n",
    "    nlp = spacy.load(\"en_core_web_md\")\n",
    "else:\n",
    "    nlp = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm_notebook\n",
    "\n",
    "from spacy_crfsuite import read_file\n",
    "from spacy_crfsuite.train import gold_example_to_crf_tokens\n",
    "from spacy_crfsuite.tokenizer import SpacyTokenizer\n",
    "\n",
    "def read_examples(file, tokenizer, use_dense_features=False, limit=None):\n",
    "    examples = []\n",
    "    it = read_file(file)\n",
    "    it = it[:limit] if limit else it\n",
    "    for raw_example in tqdm_notebook(it, desc=file):\n",
    "        crf_example = gold_example_to_crf_tokens(\n",
    "            raw_example, \n",
    "            tokenizer=tokenizer, \n",
    "            use_dense_features=use_dense_features, \n",
    "            bilou=False\n",
    "        )\n",
    "        examples.append(crf_example)\n",
    "    return examples\n",
    "\n",
    "# Spacy tokenizer\n",
    "tokenizer = SpacyTokenizer(nlp)\n",
    "\n",
    "# OPTIONAL: fine-tune hyper-params\n",
    "# this is going to take a while, so you might need a coffee break ...\n",
    "dev_examples = None\n",
    "# dev_examples = read_examples(\"conll03/valid.conll\", tokenizer, use_dense_features=use_dense_features)\n",
    "\n",
    "if dev_examples:\n",
    "    rs = crf_extractor.fine_tune(dev_examples, cv=5, n_iter=30, random_state=42)\n",
    "    print(\"best params:\", rs.best_params_, \", score:\", rs.best_score_)\n",
    "    crf_extractor.component_config.update(rs.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "514364ea909a42ccb10f2c9bebc9e476",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='conll03/train.conll', max=14041.0, style=ProgressStyle(de…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Most likely transitions:\n",
      "B-ORG      -> I-ORG      7.260039\n",
      "B-PER      -> I-PER      7.016621\n",
      "I-ORG      -> I-ORG      6.787570\n",
      "I-MISC     -> I-MISC     6.301491\n",
      "B-LOC      -> I-LOC      6.284100\n",
      "B-MISC     -> I-MISC     6.052984\n",
      "I-LOC      -> I-LOC      5.508632\n",
      "I-PER      -> I-PER      4.808802\n",
      "O          -> B-PER      3.496929\n",
      "O          -> O          2.778873\n",
      "\n",
      "Positive features:\n",
      "5.259367 O          0:bias:bias\n",
      "4.243087 O          0:suffix3:day\n",
      "3.915058 B-ORG      -1:low:v\n",
      "3.763559 B-PER      BOS\n",
      "3.526967 O          BOS\n",
      "3.258426 B-PER      0:prefix2:Mc\n",
      "3.239694 O          0:prefix2:W1\n",
      "3.103391 B-LOC      BOS\n",
      "3.099422 B-ORG      BOS\n",
      "2.878765 B-ORG      0:suffix5:shire\n"
     ]
    }
   ],
   "source": [
    "train_examples = read_examples(\"conll03/train.conll\", tokenizer=tokenizer, use_dense_features=use_dense_features)\n",
    "\n",
    "crf_extractor.train(train_examples, dev_samples=dev_examples)\n",
    "print(crf_extractor.explain())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f3c6269ab42435f88a2264dd2790164",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='conll03/test.conll', max=3453.0, style=ProgressStyle(desc…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/talmago/git/spacy_crfsuite/.venv/lib/python3.6/site-packages/sklearn/utils/validation.py:71: FutureWarning: Pass labels=['B-LOC', 'I-LOC', 'B-MISC', 'I-MISC', 'B-ORG', 'I-ORG', 'B-PER', 'I-PER'] as keyword args. From version 0.25 passing these as positional arguments will result in an error\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "       B-LOC      0.855     0.880     0.867      1615\n",
      "       I-LOC      0.742     0.714     0.728       238\n",
      "      B-MISC      0.804     0.759     0.781       698\n",
      "      I-MISC      0.665     0.671     0.668       207\n",
      "       B-ORG      0.816     0.729     0.770      1644\n",
      "       I-ORG      0.727     0.746     0.736       818\n",
      "       B-PER      0.855     0.867     0.861      1592\n",
      "       I-PER      0.887     0.960     0.922      1117\n",
      "\n",
      "   micro avg      0.826     0.822     0.824      7929\n",
      "   macro avg      0.794     0.791     0.792      7929\n",
      "weighted avg      0.825     0.822     0.823      7929\n",
      "\n"
     ]
    }
   ],
   "source": [
    "test_examples = read_examples(\"conll03/test.conll\", tokenizer=tokenizer, use_dense_features=use_dense_features)\n",
    "\n",
    "f1_score, classification_report = crf_extractor.eval(test_examples)\n",
    "print(classification_report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "George Walker Bush - PER\n",
      "American - MISC\n",
      "United States - LOC\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/talmago/git/spacy_crfsuite/spacy_crfsuite/crf_extractor.py:391: UserWarning: Inconsistent BILOU tagging found, B- tag, L- tag pair encloses multiple entity classes.i.e. [B-a, I-b, L-a] instead of [B-a, I-a, L-a].\n",
      "Assuming B- class is correct.\n",
      "  \"Inconsistent BILOU tagging found, B- tag, L- \"\n",
      "/Users/talmago/git/spacy_crfsuite/spacy_crfsuite/crf_extractor.py:408: UserWarning: Inconsistent BILOU tagging found, B- tag not closed by L- tag, i.e [B-a, I-a, O] instead of [B-a, L-a, O].\n",
      "Assuming last tag is L-\n",
      "  \"Inconsistent BILOU tagging found, B- tag not \"\n"
     ]
    }
   ],
   "source": [
    "# Example of a spaCy pipeline\n",
    "from spacy_crfsuite import CRFEntityExtractor\n",
    "\n",
    "# Add our CRF component to pipeline\n",
    "nlp = spacy.load(\"en_core_web_sm\", disable=[\"ner\"])\n",
    "pipe = CRFEntityExtractor(nlp, crf_extractor=crf_extractor)\n",
    "nlp.add_pipe(pipe)\n",
    "\n",
    "# And use natively ..\n",
    "doc = nlp(\n",
    "    \"George Walker Bush (born July 6, 1946) is an American politician and businessman \"\n",
    "    \"who served as the 43rd president of the United States from 2001 to 2009.\")\n",
    "\n",
    "for ent in doc.ents:\n",
    "    print(ent, \"-\", ent.label_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save model to disk ..\n",
    "model_name = f\"conll03_{nlp._meta['lang']}_{nlp._meta['name']}.bz2\"\n",
    "crf_extractor.to_disk(model_name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}