Untitled.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "import docx\n",
    "import spacy, os\n",
    "nlp = spacy.load(\"en\")\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Custom Named Entity Recognition Using spaCy\n",
    "https://towardsdatascience.com/custom-named-entity-recognition-using-spacy-7140ebbb3718\n",
    "\n",
    "Other\n",
    "https://www.analyticsvidhya.com/blog/2017/04/natural-language-processing-made-easy-using-spacy-%E2%80%8Bin-python/\n",
    "https://automatetheboringstuff.com/chapter13/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "STAMP DUTY KARNATAKA\n",
      "c$j5Coz3f$ 3oc>rtjs\n",
      "s&o^c^ost\n",
      "^C7e)\n",
      "©\n",
      "c> &sd: &&.\n",
      "2/\n",
      "ds\tafss&de dTOjSczSrt eroaiaieef«^*^x\n",
      "This sheet can be used for any document _ . , ,\n",
      "rossssSrfrS d,\n",
      "WR E N T A L AGREEMENT”\n",
      "This Rental Agreement is made and Executed on\n",
      "at Bangalore, by and Between:- wef 1/4/2014\n",
      "Mr .Giddappa\n",
      "S/o lute Nagappa\n",
      "D No.134 2nd cross\n",
      "Shunthi L/o Rumamurthynagar\n",
      "Bangalore 560016\n",
      "Hereinafter Called the \"OWNER\" of the One Part:\n",
      "Ms Pottumurthi Udayalaxmi\n",
      "W/o Ramakr ishna Kajjam\n",
      "H No.27-7 Main Bazar,\n",
      "Miryalaguda, i,\n",
      "Nalgonda Dist A p 503207.\n",
      "Hereinafter called the \"TENANT\" of the Other Part witnesseth as follows terms and conditions.\n",
      "1.\n",
      "Whereas the Tenant has paid Advance amount of Rs.40,000/-\n",
      "( Forty thousand only\t).\n",
      "; to the owner.\n",
      "This amount shall not carry any interest and this is refundable at the time vacating..\n",
      "If any arrears balance that charges will deducted from the advance amount at the time of vacating.\n",
      "has agreed to pay the Rental Charges of thousand three hundred only\t) t<\n",
      "month regularly.\n",
      "the Electrcity charges separately to\n",
      "o\n",
      "\n",
      "4f\n",
      "This Rental Agreement Duration for only 11(Eleven) months from this date of Agreement.\n",
      "If both the parties mutual understanding it can be extended.\n",
      "The Tenancy can be terminated by either party by giving Two Month notice to the opposite party.\n",
      "Whereas the Tenant shall not make any addition?!\n",
      "alteration to the schedule premises and tenant keep the portion clean and good tenantable conditions.\n",
      "Whereas the Tenant shall use the Schedule premises for Residential purpose only and should not use for any illegal or immoral purposes.\n",
      "Whereas the Tenant should pay painting charges at the time vacating the premises.\n",
      "S C H E D u L E\"\n",
      "The House premises situated at No. 184\n",
      "1st floor 2nd cross\n",
      "ohanchi h/o, Ruma.murth.yne.gar.\n",
      "Bangalore 560016 consists of 1 BHK. with bathroom & toilet.\n",
      "In witness whereof the party is hereto have signed this Rental Agreement on the day,month and year first above written in front of the following witnesses.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "sampleObj = [{\n",
    "    \"id\": 5,\n",
    "    \"paragraphs\": [{\n",
    "        \"raw\": \"ss\",\n",
    "        \"sentences\": [{\n",
    "            \"tokens\": [{\n",
    "                \"id\": 5,\n",
    "                \"dep\": \"ss\",\n",
    "                \"head\": 5,\n",
    "                \"tag\": \"ss\",\n",
    "                \"orth\": \"ss\",\n",
    "                \"ner\": \"ss\"     \n",
    "            }],\n",
    "            \"brackets\": [{\n",
    "                \"first\": 5,\n",
    "                \"last\": 5, \n",
    "                \"label\": \"ss\"}]\n",
    "        }]\n",
    "    }]\n",
    "}]\n",
    "\n",
    "\n",
    "\n",
    "df = pd.read_csv(os.getcwd() + \"\\\\..\\\\\"+  \"TrainingTestSet.csv\")\n",
    "\n",
    "docID = 0\n",
    "for file in os.listdir(os.getcwd() + \"\\\\..\\\\data\\\\training\\\\\") :\n",
    "    doc = docx.Document(os.getcwd() + \"\\\\..\\\\data\\\\training\\\\\"+  file)\n",
    "    paragraphsInThisDocument = [x.text for x in do.paragraphs]\n",
    "    docID = docID + 1\n",
    "    documentObj = {\"id\": docID}\n",
    "    \n",
    "    paraID = 0\n",
    "    for para in do.paragraphs:\n",
    "        paraID = paraID + 1\n",
    "        paraObj = {\"id\": paraID}\n",
    "        sentencesInThisPara = [s.text for s in nlp(para).sents]\n",
    "        \n",
    "        sentenceID = 0\n",
    "        for s in sentencesInThisPara:\n",
    "            sentenceID = sentenceID + 1\n",
    "            sentenceObj = {\"id\": sentenceID}\n",
    "            \n",
    "    \n",
    "\n",
    "\n",
    "\n",
    "import json\n",
    "zz = json.dumps(xx)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setting up the pipeline and entity recognizer.\n",
    "if model is not None:\n",
    "    nlp = spacy.load(model)  # load existing spacy model\n",
    "    print(\"Loaded model '%s'\" % model)\n",
    "else:\n",
    "    nlp = spacy.blank('en')  # create blank Language class\n",
    "    print(\"Created blank 'en' model\")\n",
    "if 'ner' not in nlp.pipe_names:\n",
    "    ner = nlp.create_pipe('ner')\n",
    "    nlp.add_pipe(ner)\n",
    "else:\n",
    "    ner = nlp.get_pipe('ner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add new entity labels to entity recognizer\n",
    "for i in LABEL:\n",
    "    ner.add_label(i)\n",
    "# Inititalizing optimizer\n",
    "if model is None:\n",
    "    optimizer = nlp.begin_training()\n",
    "else:\n",
    "    optimizer = nlp.entity.create_optimizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get names of other pipes to disable them during training to train # only NER and update the weights\n",
    "other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']\n",
    "with nlp.disable_pipes(*other_pipes):  # only train NER\n",
    "    for itn in range(n_iter):\n",
    "        random.shuffle(TRAIN_DATA)\n",
    "        losses = {}\n",
    "        batches = minibatch(TRAIN_DATA, \n",
    "                            size=compounding(4., 32., 1.001))\n",
    "        for batch in batches:\n",
    "            texts, annotations = zip(*batch) \n",
    "            # Updating the weights\n",
    "            nlp.update(texts, annotations, sgd=optimizer, \n",
    "                       drop=0.35, losses=losses)\n",
    "        print('Losses', losses)\n",
    "            nlp.update(texts, annotations, sgd=optimizer, \n",
    "                       drop=0.35, losses=losses)\n",
    "        print('Losses', losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save model \n",
    "if output_dir is not None:\n",
    "    output_dir = Path(output_dir)\n",
    "    if not output_dir.exists():\n",
    "        output_dir.mkdir()\n",
    "    nlp.meta['name'] = new_model_name  # rename model\n",
    "    nlp.to_disk(output_dir)\n",
    "    print(\"Saved model to\", output_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test the saved model\n",
    "print(\"Loading from\", output_dir)\n",
    "nlp2 = spacy.load(output_dir)\n",
    "doc2 = nlp2(test_text)\n",
    "for ent in doc2.ents:\n",
    "    print(ent.label_, ent.text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}