PoliticalBiasnessDetection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Importing Packages\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''If packages are not found then use: pip install package_name'''\n",
    "import pandas as pd\n",
    "import re\n",
    "import string\n",
    "from textblob import TextBlob\n",
    "import nltk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data pre processing and data cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(225, 6)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class dataPreProcessing:\n",
    "    def __init__(self,path):\n",
    "        self.data=pd.read_csv(path,delimiter = \",\",encoding='ISO-8859–1')\n",
    "        self.data.columns = ((self.data.columns.str).replace(\"^ \",\"\")).str.replace(\" $\",\"\")   #removing cloumns spaces\n",
    "        self.data[\"Article\"].fillna(\"To be added\", inplace = True)\n",
    "        self.data[\"Headline\"].fillna(\"To be added\", inplace = True)       #replacing NaN values with some text\n",
    "        self.data[\"Label\"].fillna(\"To be added\", inplace = True)\n",
    "        self.data[\"Publisher\"].fillna(\"To be added\", inplace = True)\n",
    "    def read_data(self):\n",
    "        return self.data\n",
    "    def clean_article(self,article):\n",
    "        article=article.lower()\n",
    "        article = re.sub('\\[.*?\\]', '', article)\n",
    "        article = re.sub('[%s]' % re.escape(string.punctuation), '', article)\n",
    "        article = re.sub('\\w*\\d\\w*', '', article)\n",
    "        return article\n",
    "    def clean_dataset(self):\n",
    "        for i in range(len(self.data)):\n",
    "            self.data['Article'][i]=self.clean_article(self.data['Article'][i])\n",
    "            self.data['Headline'][i]=self.clean_article(self.data['Headline'][i])\n",
    "            self.data['Label'][i]=self.data['Label'][i].lower()\n",
    "        return self.data\n",
    "    def tokenize(self):\n",
    "        pass\n",
    "    def remove_stopwords(self):\n",
    "        pass\n",
    "        \n",
    "data = dataPreProcessing('data/Data3.csv')\n",
    "data.read_data().shape           # for comparison of articles\n",
    "#data.read_data().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:19: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in a momentous decision government moved parliament yesterday to abate article  of the constitution and supersede article  thereby effectively ceasing the special status for jammu  kashmir simultaneously union home minister amit shah introduced the jammu and kashmir reorganisation bill to bifurcate the state and create two separate union territories  jammu  kashmir with a legislature and ladakh without a legislature this means that the constitution of india shall now apply in full force across jk and ladakh and the jk legislature will henceforth not have any discretionary powers to decide who permanent residents of the territory are or provide them with special rights with respect to employment and acquisition of property\r\n",
      "abrogating article  has been a longheld wish of bjp which sees the provision as a historical wrong however the complicated history of kashmir and the sensitivity surrounding the matter had hitherto prevented such a move but with the current bjp government projecting muscular nationalism and enjoying significant numerical strength in parliament cessation of article  became more doable given that abrogating article  was part of the bjp manifesto on the basis of which it swept lok sabha polls the government has the mandate to do this that nonnda parties like bsp aap ysrcp bjd and aiadmk too have come out in support of the governments move shows which way the winds are blowing\r\n",
      "arguably article  had contributed to the problems in kashmir the states special powers of autonomy meant in practice arrogation of power by a tiny valley elite which led to a feeling of not being represented among the people of jammu and ladakh as well as disaffection feeding into separatist sentiment in the valley itself and given that kashmir came to be a troubled state with separatist sentiments being stoked by pakistan with the help of crossborder terrorism this may have been the worst of both worlds besides article  was meant to be a temporary provision in that sense its cessation now is congruent with historical constitutional perspective\r\n",
      "that said there are some issues with the way in which article  is being nullified and the state bifurcated even according to the constitutional provision to abate article  consultation with the jk state assembly is deemed necessary before a constitutional order to this effect is passed the government however has chosen to use the fact that there is no jk government at the moment and substituted the latter with the jk governor while this approach may or may not be legally valid it certainly bypasses wider discussion on the matter that would have been democratically sound\r\n",
      "second the logic behind converting the state into two union territories is also not plain true there has been a demand for union territory status in ladakh for some time but with the cessation of article  the special powers accruing to the jk assembly in srinagar would also extinguish thus it is debatable if bifurcation of the state and creation of the union territory of ladakh was necessary after that\r\n",
      "in fact this would be the first time in the history of india that a state was being converted into union territories  the trend has been the other way around hence given the magnitude of the decision more allround discussion would have been prudent but perhaps governments decision to push ahead with its article  move was linked to external factors with the us looking to wind down its military engagement in afghanistan and turning to pakistan to facilitate a peace deal with afghan taliban islamabad has been upping the ante on kashmir\r\n",
      "new delhi perhaps felt that islamabad was gaining too much leverage in the emerging regional scenario and had to do something to counter this whatever the compulsions even if article  is abrogated new delhi will have to  sooner rather than later  grant full statehood rights to jammu  kashmir and perhaps ladakh too for the sake of democratic governance as well as the federal principles in which india is grounded\n"
     ]
    }
   ],
   "source": [
    "df=data.clean_dataset()['Article'][1]    #for comparison of articles\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenizing, Lemmatizing, and removing stop-words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\User\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:19: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "as the ram temple gets under way india must put the past of a communal struggle behind\n",
      "the bhoomi pujan or the groundbreaking ceremony for the construction of a grand temple for lord sri ram in ayodhya on wednesday marks an end and a beginning what it ends and what it begins can both be interpreted in different ways how india collectively makes meaning out of it will define the future of the country hereon one view is that the rising ram temple signifies the end of perceived humiliation of the hindus and the beginning of a new phase of their political ascendancy the other is that it denotes the end of strife that shackled indias potential for decades and heralds a new dawn of fraternity among religious communities the end and the beginning therefore are not just open to interpretation they hold the possibilities of change for those who yearned for a temple at the site which they believe is the exact spot of sri rams birth the journey so far has been tumultuous and violent a muslim place of worship that stood there for  years was demolished in  to make way for the temple  a serious crime according to the supreme court order last year that handed over the site to the hindus the proponents of the temple must consider this an occasion to seek conciliation over conquest dialogue over diatribe and tranquility over triumphalism\n",
      " \n",
      "the ceremony itself manifested multiple possibilities for the countrys future in symbolism and rhetoric the line of separation between state and religion was ominously crossed notably by the role of prime minister narendra modi in it in his speech however he cited lord rams adherence to justice fairness and empathy for the vulnerable he emphasised the importance of these values for the present but while outlining a road map for an inclusive future his interpretation of the past echoed familiar tropes of sectarian politics relitigating historical wrongs has rarely been the foundation for a harmonious and prosperous future in indias case many of them are an outcome of its unpleasant encounter with british colonialism recent pathbreaking studies in genetics have unearthed indias past of being a melting pot of populations and cultures over millennia india must put the acrimonious political mobilisations over religious issues behind it and look forward to modern secular governance the construction of the temple is the logical result of the supreme court judgment it should mark the end of an older bitter phase of india and the beginning of a new harmonious phase\n",
      "424\n",
      "['ram', 'temple', 'gets', 'way', 'india', 'must', 'put', 'past', 'communal', 'struggle', 'behind', 'bhoomi', 'pujan', 'groundbreaking', 'ceremony', 'construction', 'grand', 'temple', 'lord', 'sri', 'ram', 'ayodhya', 'wednesday', 'marks', 'end', 'beginning', 'ends', 'begins', 'interpreted', 'different', 'ways', 'india', 'collectively', 'makes', 'meaning', 'define', 'future', 'country', 'hereon', 'one', 'view', 'rising', 'ram', 'temple', 'signifies', 'end', 'perceived', 'humiliation', 'hindus', 'beginning', 'new', 'phase', 'political', 'ascendancy', 'denotes', 'end', 'strife', 'shackled', 'indias', 'potential', 'decades', 'heralds', 'new', 'dawn', 'fraternity', 'among', 'religious', 'communities', 'end', 'beginning', 'therefore', 'open', 'interpretation', 'hold', 'possibilities', 'change', 'yearned', 'temple', 'site', 'believe', 'exact', 'spot', 'sri', 'rams', 'birth', 'journey', 'far', 'tumultuous', 'violent', 'muslim', 'place', 'worship', 'stood', 'years', 'demolished', 'make', 'way', 'temple', 'serious', 'crime', 'according', 'supreme', 'court', 'order', 'last', 'year', 'handed', 'site', 'hindus', 'proponents', 'temple', 'must', 'consider', 'occasion', 'seek', 'conciliation', 'conquest', 'dialogue', 'diatribe', 'tranquility', 'triumphalism', 'ceremony', 'manifested', 'multiple', 'possibilities', 'countrys', 'future', 'symbolism', 'rhetoric', 'line', 'separation', 'state', 'religion', 'ominously', 'crossed', 'notably', 'role', 'prime', 'minister', 'narendra', 'modi', 'speech', 'however', 'cited', 'lord', 'rams', 'adherence', 'justice', 'fairness', 'empathy', 'vulnerable', 'emphasised', 'importance', 'values', 'present', 'outlining', 'road', 'map', 'inclusive', 'future', 'interpretation', 'past', 'echoed', 'familiar', 'tropes', 'sectarian', 'politics', 'relitigating', 'historical', 'wrongs', 'rarely', 'foundation', 'harmonious', 'prosperous', 'future', 'indias', 'case', 'many', 'outcome', 'unpleasant', 'encounter', 'british', 'colonialism', 'recent', 'pathbreaking', 'studies', 'genetics', 'unearthed', 'indias', 'past', 'melting', 'pot', 'populations', 'cultures', 'millennia', 'india', 'must', 'put', 'acrimonious', 'political', 'mobilisations', 'religious', 'issues', 'behind', 'look', 'forward', 'modern', 'secular', 'governance', 'construction', 'temple', 'logical', 'result', 'supreme', 'court', 'judgment', 'mark', 'end', 'older', 'bitter', 'phase', 'india', 'beginning', 'new', 'harmonious', 'phase']\n",
      "226\n",
      "['ram', 'temple', 'get', 'way', 'india', 'must', 'put', 'past', 'communal', 'struggle', 'behind', 'bhoomi', 'pujan', 'groundbreaking', 'ceremony', 'construction', 'grand', 'temple', 'lord', 'sri', 'ram', 'ayodhya', 'wednesday', 'mark', 'end', 'begin', 'end', 'begin', 'interpret', 'different', 'ways', 'india', 'collectively', 'make', 'mean', 'define', 'future', 'country', 'hereon', 'one', 'view', 'rise', 'ram', 'temple', 'signify', 'end', 'perceive', 'humiliation', 'hindus', 'begin', 'new', 'phase', 'political', 'ascendancy', 'denote', 'end', 'strife', 'shackle', 'indias', 'potential', 'decades', 'herald', 'new', 'dawn', 'fraternity', 'among', 'religious', 'communities', 'end', 'begin', 'therefore', 'open', 'interpretation', 'hold', 'possibilities', 'change', 'yearn', 'temple', 'site', 'believe', 'exact', 'spot', 'sri', 'ram', 'birth', 'journey', 'far', 'tumultuous', 'violent', 'muslim', 'place', 'worship', 'stand', 'years', 'demolish', 'make', 'way', 'temple', 'serious', 'crime', 'accord', 'supreme', 'court', 'order', 'last', 'year', 'hand', 'site', 'hindus', 'proponents', 'temple', 'must', 'consider', 'occasion', 'seek', 'conciliation', 'conquest', 'dialogue', 'diatribe', 'tranquility', 'triumphalism', 'ceremony', 'manifest', 'multiple', 'possibilities', 'countrys', 'future', 'symbolism', 'rhetoric', 'line', 'separation', 'state', 'religion', 'ominously', 'cross', 'notably', 'role', 'prime', 'minister', 'narendra', 'modi', 'speech', 'however', 'cite', 'lord', 'ram', 'adherence', 'justice', 'fairness', 'empathy', 'vulnerable', 'emphasise', 'importance', 'value', 'present', 'outline', 'road', 'map', 'inclusive', 'future', 'interpretation', 'past', 'echo', 'familiar', 'tropes', 'sectarian', 'politics', 'relitigating', 'historical', 'wrong', 'rarely', 'foundation', 'harmonious', 'prosperous', 'future', 'indias', 'case', 'many', 'outcome', 'unpleasant', 'encounter', 'british', 'colonialism', 'recent', 'pathbreaking', 'study', 'genetics', 'unearth', 'indias', 'past', 'melt', 'pot', 'populations', 'culture', 'millennia', 'india', 'must', 'put', 'acrimonious', 'political', 'mobilisations', 'religious', 'issue', 'behind', 'look', 'forward', 'modern', 'secular', 'governance', 'construction', 'temple', 'logical', 'result', 'supreme', 'court', 'judgment', 'mark', 'end', 'older', 'bitter', 'phase', 'india', 'begin', 'new', 'harmonious', 'phase']\n"
     ]
    }
   ],
   "source": [
    "#tokenize, lemmatize, remove stop-words\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "nltk.download('stopwords')\n",
    "\n",
    "\n",
    "df=data.clean_dataset()['Article'][45]    #for comparison of articles\n",
    "print(df)\n",
    "\n",
    "tokens=word_tokenize(df)\n",
    "#print(tokens)\n",
    "wordsInArticle= len(tokens)\n",
    "print(wordsInArticle)\n",
    "\n",
    "def remove_stop_words(tokens):\n",
    "    stopwordsInArticle=[]\n",
    "    stopwordsNotInArticle=[]\n",
    "    for i in tokens:\n",
    "        if i not in stopwords.words('english'):\n",
    "            stopwordsNotInArticle.append(i)\n",
    "        else:\n",
    "            stopwordsInArticle.append(i)\n",
    "    return stopwordsInArticle,stopwordsNotInArticle\n",
    "\n",
    "stopwordsInArticle,stopwordsNotInArticle=remove_stop_words(tokens)\n",
    "print(stopwordsNotInArticle)\n",
    "\n",
    "wordsInArticle= len(stopwordsNotInArticle)\n",
    "print(wordsInArticle)\n",
    "\n",
    "def lemmatise(tokens):\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    for i in range(len(tokens)):\n",
    "        tokens[i]=lemmatizer.lemmatize(tokens[i],\"v\")\n",
    "    return tokens\n",
    "\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "for i in range(len(stopwordsNotInArticle)):\n",
    "    stopwordsNotInArticle[i]=lemmatizer.lemmatize(stopwordsNotInArticle[i],\"v\")    #reduce the word to its infinitive form\n",
    "print(stopwordsNotInArticle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:19: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['momentous', 'decision', 'government', 'move', 'parliament', 'yesterday', 'abate', 'article', 'constitution', 'supersede', 'article', 'thereby', 'effectively', 'cease', 'special', 'status', 'jammu', 'kashmir', 'simultaneously', 'union', 'home', 'minister', 'amit', 'shah', 'introduce', 'jammu', 'kashmir', 'reorganisation', 'bill', 'bifurcate', 'state', 'create', 'two', 'separate', 'union', 'territories', 'jammu', 'kashmir', 'legislature', 'ladakh', 'without', 'legislature', 'mean', 'constitution', 'india', 'shall', 'apply', 'full', 'force', 'across', 'jk', 'ladakh', 'jk', 'legislature', 'henceforth', 'discretionary', 'power', 'decide', 'permanent', 'residents', 'territory', 'provide', 'special', 'right', 'respect', 'employment', 'acquisition', 'property', 'abrogate', 'article', 'longheld', 'wish', 'bjp', 'see', 'provision', 'historical', 'wrong', 'however', 'complicate', 'history', 'kashmir', 'sensitivity', 'surround', 'matter', 'hitherto', 'prevent', 'move', 'current', 'bjp', 'government', 'project', 'muscular', 'nationalism', 'enjoy', 'significant', 'numerical', 'strength', 'parliament', 'cessation', 'article', 'become', 'doable', 'give', 'abrogate', 'article', 'part', 'bjp', 'manifesto', 'basis', 'sweep', 'lok', 'sabha', 'poll', 'government', 'mandate', 'nonnda', 'party', 'like', 'bsp', 'aap', 'ysrcp', 'bjd', 'aiadmk', 'come', 'support', 'governments', 'move', 'show', 'way', 'wind', 'blow', 'arguably', 'article', 'contribute', 'problems', 'kashmir', 'state', 'special', 'power', 'autonomy', 'mean', 'practice', 'arrogation', 'power', 'tiny', 'valley', 'elite', 'lead', 'feel', 'represent', 'among', 'people', 'jammu', 'ladakh', 'well', 'disaffection', 'feed', 'separatist', 'sentiment', 'valley', 'give', 'kashmir', 'come', 'trouble', 'state', 'separatist', 'sentiments', 'stoke', 'pakistan', 'help', 'crossborder', 'terrorism', 'may', 'worst', 'worlds', 'besides', 'article', 'mean', 'temporary', 'provision', 'sense', 'cessation', 'congruent', 'historical', 'constitutional', 'perspective', 'say', 'issue', 'way', 'article', 'nullify', 'state', 'bifurcate', 'even', 'accord', 'constitutional', 'provision', 'abate', 'article', 'consultation', 'jk', 'state', 'assembly', 'deem', 'necessary', 'constitutional', 'order', 'effect', 'pass', 'government', 'however', 'choose', 'use', 'fact', 'jk', 'government', 'moment', 'substitute', 'latter', 'jk', 'governor', 'approach', 'may', 'may', 'legally', 'valid', 'certainly', 'bypass', 'wider', 'discussion', 'matter', 'would', 'democratically', 'sound', 'second', 'logic', 'behind', 'convert', 'state', 'two', 'union', 'territories', 'also', 'plain', 'true', 'demand', 'union', 'territory', 'status', 'ladakh', 'time', 'cessation', 'article', 'special', 'power', 'accrue', 'jk', 'assembly', 'srinagar', 'would', 'also', 'extinguish', 'thus', 'debatable', 'bifurcation', 'state', 'creation', 'union', 'territory', 'ladakh', 'necessary', 'fact', 'would', 'first', 'time', 'history', 'india', 'state', 'convert', 'union', 'territories', 'trend', 'way', 'around', 'hence', 'give', 'magnitude', 'decision', 'allround', 'discussion', 'would', 'prudent', 'perhaps', 'governments', 'decision', 'push', 'ahead', 'article', 'move', 'link', 'external', 'factor', 'us', 'look', 'wind', 'military', 'engagement', 'afghanistan', 'turn', 'pakistan', 'facilitate', 'peace', 'deal', 'afghan', 'taliban', 'islamabad', 'up', 'ante', 'kashmir', 'new', 'delhi', 'perhaps', 'felt', 'islamabad', 'gain', 'much', 'leverage', 'emerge', 'regional', 'scenario', 'something', 'counter', 'whatever', 'compulsions', 'even', 'article', 'abrogate', 'new', 'delhi', 'sooner', 'rather', 'later', 'grant', 'full', 'statehood', 'right', 'jammu', 'kashmir', 'perhaps', 'ladakh', 'sake', 'democratic', 'governance', 'well', 'federal', 'principles', 'india', 'ground']\n"
     ]
    }
   ],
   "source": [
    "article=data.clean_dataset()['Article'][1]\n",
    "tokens=word_tokenize(article)\n",
    "stopwordsInArticle,stopwordsNotInArticle=remove_stop_words(tokens)\n",
    "stopwordsNotInArticle=lemmatise(stopwordsNotInArticle)\n",
    "print(stopwordsNotInArticle)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:19: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'article': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224], 'bjp_count': [0.024489795918367346, 0.05027932960893855, 0.0326975476839237, 0.03980099502487562, 0.02823529411764706, 0.06072106261859583, 0.04878048780487805, 0.0297029702970297, 0.06467661691542288, 0.021739130434782608, 0.012552301255230125, 0.014059753954305799, 0.018494055482166448, 0.015267175572519083, 0.027196652719665274, 0.01171875, 0.026041666666666668, 0.0163265306122449, 0.0, 0.008298755186721992, 0.11297071129707113, 0.06694560669456066, 0.02681992337164751, 0.056372549019607844, 0.04585152838427948, 0.02494802494802495, 0.06944444444444445, 0.045871559633027525, 0.04477611940298507, 0.007936507936507936, 0.008849557522123894, 0.007722007722007722, 0.04032258064516129, 0.014814814814814815, 0.013289036544850499, 0.0, 0.014778325123152709, 0.07450980392156863, 0.027777777777777776, 0.023054755043227664, 0.048, 0.017543859649122806, 0.006, 0.0425531914893617, 0.046948356807511735, 0.035398230088495575, 0.03496503496503497, 0.05470459518599562, 0.05394190871369295, 0.030716723549488054, 0.03111111111111111, 0.038560411311053984, 0.032036613272311214, 0.0076045627376425855, 0.017391304347826087, 0.011884550084889643, 0.004379562043795621, 0.021645021645021644, 0.024, 0.0182648401826484, 0.014150943396226415, 0.01932367149758454, 0.05058365758754864, 0.046511627906976744, 0.055710306406685235, 0.03349282296650718, 0.007352941176470588, 0.005244755244755245, 0.054404145077720206, 0.0196078431372549, 0.013953488372093023, 0.033426183844011144, 0.04460966542750929, 0.028708133971291867, 0.07906976744186046, 0.05627705627705628, 0.016666666666666666, 0.04492512479201331, 0.0273972602739726, 0.02702702702702703, 0.08333333333333333, 0.018957345971563982, 0.05405405405405406, 0.04430379746835443, 0.029333333333333333, 0.10059171597633136, 0.049079754601226995, 0.03207547169811321, 0.10457516339869281, 0.04504504504504504, 0.04697986577181208, 0.08076923076923077, 0.1033210332103321, 0.04721030042918455, 0.03888888888888889, 0.030201342281879196, 0.0273972602739726, 0.15425531914893617, 0.052873563218390804, 0.04291845493562232, 0.05714285714285714, 0.07971014492753623, 0.0213903743315508, 0.01834862385321101, 0.011111111111111112, 0.06280193236714976, 0.08275862068965517, 0.041666666666666664, 0.06367041198501873, 0.008130081300813009, 0.07177033492822966, 0.017766497461928935, 0.040123456790123455, 0.08144796380090498, 0.14516129032258066, 0.0243161094224924, 0.03139013452914798, 0.05027932960893855, 0.007722007722007722, 0.09019607843137255, 0.10828025477707007, 0.06285714285714286, 0.15833333333333333, 0.04987531172069826, 0.09701492537313433, 0.1469387755102041, 0.005154639175257732, 0.07142857142857142, 0.0106951871657754, 0.0, 0.03272727272727273, 0.06716417910447761, 0.04245283018867924, 0.05263157894736842, 0.010638297872340425, 0.059880239520958084, 0.06013745704467354, 0.05725190839694656, 0.06172839506172839, 0.07412790697674419, 0.012121212121212121, 0.011928429423459244, 0.05339805825242718, 0.0962962962962963, 0.05172413793103448, 0.030223390275952694, 0.03940886699507389, 0.02386634844868735, 0.09803921568627451, 0.02531645569620253, 0.06278026905829596, 0.036613272311212815, 0.1348314606741573, 0.01694915254237288, 0.1223529411764706, 0.034636871508379886, 0.06698564593301436, 0.04987531172069826, 0.10457516339869281, 0.04827586206896552, 0.04468085106382979, 0.09239130434782608, 0.14615384615384616, 0.08322496749024708, 0.04582210242587601, 0.046753246753246755, 0.027624309392265192, 0.02857142857142857, 0.04040404040404041, 0.058004640371229696, 0.027450980392156862, 0.014814814814814815, 0.05555555555555555, 0.09482758620689655, 0.07770961145194274, 0.027131782945736434, 0.032407407407407406, 0.012012012012012012, 0.014234875444839857, 0.010101010101010102, 0.0038910505836575876, 0.019830028328611898, 0.021739130434782608, 0.016483516483516484, 0.006172839506172839, 0.0, 0.012285012285012284, 0.014028056112224449, 0.0, 0.003316749585406302, 0.011049723756906077, 0.003355704697986577, 0.009345794392523364, 0.0, 0.010752688172043012, 0.01524390243902439, 0.03006012024048096, 0.0, 0.013245033112582781, 0.012345679012345678, 0.009554140127388535, 0.027777777777777776, 0.0041841004184100415, 0.008658008658008658, 0.018970189701897018, 0.0, 0.0196078431372549, 0.03260869565217391, 0.03333333333333333, 0.010869565217391304, 0.011834319526627219, 0.018404907975460124, 0.0038314176245210726, 0.013186813186813187, 0.0020408163265306124, 0.011661807580174927, 0.0121580547112462, 0.0, 0.013618677042801557, 0.019823788546255508, 0.03197158081705151, 0.018656716417910446, 0.026570048309178744, 0.004694835680751174, 0.0017211703958691911]}\n"
     ]
    }
   ],
   "source": [
    "def bjp(string):\n",
    "    \n",
    "    file=open(\"data/bias-lexicon/BJP Corpus.txt\",\"r\")\n",
    "\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        for word in words:\n",
    "            if word == string:\n",
    "                #print(\"found\")\n",
    "                #print(words[0] , words[2])\n",
    "               \n",
    "                return True\n",
    "    return False\n",
    "    file.close()\n",
    "\n",
    "\n",
    "count_bjp={\n",
    "    \"article\":[],\n",
    "    \"bjp_count\":[]\n",
    "}\n",
    "size=data.read_data().shape[0]\n",
    "for j in range(size):\n",
    "    count=0\n",
    "    #data.clean_dataset()['Article'][i]\n",
    "    article=data.clean_dataset()['Article'][j]\n",
    "    tokens=word_tokenize(article)\n",
    "    stopwordsInArticle,stopwordsNotInArticle=remove_stop_words(tokens)\n",
    "    stopwordsNotInArticle=lemmatise(stopwordsNotInArticle)\n",
    "    for i in range(len(stopwordsNotInArticle)):\n",
    "        length=len(stopwordsNotInArticle)\n",
    "        result=bjp(stopwordsNotInArticle[i])\n",
    "        if(result==True):\n",
    "            count=count+1\n",
    "    count_bjp['article'].append(j)\n",
    "    count_bjp['bjp_count'].append(count/length)\n",
    "print(count_bjp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:19: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'article': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224], 'congress_count': [0.014285714285714285, 0.0111731843575419, 0.008174386920980926, 0.008291873963515755, 0.01411764705882353, 0.0056925996204933585, 0.011086474501108648, 0.009900990099009901, 0.009950248756218905, 0.021739130434782608, 0.0041841004184100415, 0.01054481546572935, 0.0066050198150594455, 0.003816793893129771, 0.029288702928870293, 0.0078125, 0.005208333333333333, 0.00816326530612245, 0.003875968992248062, 0.0, 0.0502092050209205, 0.02510460251046025, 0.007662835249042145, 0.022058823529411766, 0.013100436681222707, 0.010395010395010396, 0.015873015873015872, 0.0, 0.009950248756218905, 0.0, 0.01327433628318584, 0.007722007722007722, 0.016129032258064516, 0.0, 0.0, 0.01876675603217158, 0.0, 0.00392156862745098, 0.016666666666666666, 0.008645533141210375, 0.012, 0.02046783625730994, 0.014, 0.014184397163120567, 0.046948356807511735, 0.02654867256637168, 0.026223776223776224, 0.0262582056892779, 0.02074688796680498, 0.03754266211604096, 0.03333333333333333, 0.038560411311053984, 0.04805491990846682, 0.019011406844106463, 0.034782608695652174, 0.01697792869269949, 0.02335766423357664, 0.004329004329004329, 0.016, 0.0182648401826484, 0.018867924528301886, 0.004830917874396135, 0.0, 0.013953488372093023, 0.011142061281337047, 0.019138755980861243, 0.0, 0.0, 0.023316062176165803, 0.0, 0.0, 0.002785515320334262, 0.011152416356877323, 0.028708133971291867, 0.013953488372093023, 0.025974025974025976, 0.016666666666666666, 0.011647254575707155, 0.0182648401826484, 0.02702702702702703, 0.06666666666666667, 0.023696682464454975, 0.007722007722007722, 0.0, 0.024, 0.0, 0.024539877300613498, 0.020754716981132074, 0.017429193899782137, 0.009009009009009009, 0.03355704697986577, 0.015384615384615385, 0.01845018450184502, 0.07296137339055794, 0.022222222222222223, 0.040268456375838924, 0.0, 0.0425531914893617, 0.029885057471264367, 0.004291845493562232, 0.02857142857142857, 0.010869565217391304, 0.053475935828877004, 0.13761467889908258, 0.06111111111111111, 0.043478260869565216, 0.07586206896551724, 0.03571428571428571, 0.013108614232209739, 0.024390243902439025, 0.023923444976076555, 0.04314720812182741, 0.024691358024691357, 0.02262443438914027, 0.024193548387096774, 0.0121580547112462, 0.013452914798206279, 0.0223463687150838, 0.007722007722007722, 0.054901960784313725, 0.03821656050955414, 0.045714285714285714, 0.05, 0.032418952618453865, 0.033582089552238806, 0.0163265306122449, 0.020618556701030927, 0.05194805194805195, 0.0748663101604278, 0.013513513513513514, 0.01090909090909091, 0.03731343283582089, 0.03459119496855346, 0.042105263157894736, 0.06914893617021277, 0.04790419161676647, 0.030927835051546393, 0.011450381679389313, 0.030864197530864196, 0.046511627906976744, 0.024242424242424242, 0.02385685884691849, 0.02669902912621359, 0.05185185185185185, 0.040229885057471264, 0.04336399474375821, 0.07881773399014778, 0.07398568019093078, 0.0196078431372549, 0.0031645569620253164, 0.04932735426008968, 0.032036613272311214, 0.02247191011235955, 0.05084745762711865, 0.03058823529411765, 0.013407821229050279, 0.028708133971291867, 0.032418952618453865, 0.017429193899782137, 0.027586206896551724, 0.00851063829787234, 0.06521739130434782, 0.038461538461538464, 0.015604681404421327, 0.0431266846361186, 0.046753246753246755, 0.011049723756906077, 0.014285714285714285, 0.015151515151515152, 0.02088167053364269, 0.027450980392156862, 0.037037037037037035, 0.012345679012345678, 0.02586206896551724, 0.02044989775051125, 0.027131782945736434, 0.02546296296296296, 0.036036036036036036, 0.03202846975088968, 0.05555555555555555, 0.0038910505836575876, 0.0056657223796034, 0.004347826086956522, 0.01098901098901099, 0.006172839506172839, 0.0056179775280898875, 0.014742014742014743, 0.01002004008016032, 0.0, 0.0, 0.016574585635359115, 0.006711409395973154, 0.003115264797507788, 0.0, 0.010752688172043012, 0.013719512195121951, 0.01002004008016032, 0.018292682926829267, 0.08609271523178808, 0.037037037037037035, 0.0031847133757961785, 0.0, 0.0041841004184100415, 0.017316017316017316, 0.005420054200542005, 0.0, 0.014005602240896359, 0.002717391304347826, 0.0, 0.0036231884057971015, 0.0029585798816568047, 0.0, 0.0038314176245210726, 0.0, 0.0, 0.0029154518950437317, 0.00303951367781155, 0.0, 0.0019455252918287938, 0.013215859030837005, 0.023090586145648313, 0.011194029850746268, 0.00966183574879227, 0.002347417840375587, 0.013769363166953529]}\n"
     ]
    }
   ],
   "source": [
    "#function for congress\n",
    "\n",
    "def congress(string):\n",
    "    \n",
    "    file=open(\"data/bias-lexicon/congress-corpus.txt\",\"r\")\n",
    "\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        for word in words:\n",
    "            if word == string:\n",
    "                #print(\"found\")\n",
    "                #print(words[0] , words[2])\n",
    "               \n",
    "                return True\n",
    "    return False\n",
    "    file.close()\n",
    "\n",
    "\n",
    "count_congress={\n",
    "    \"article\":[],\n",
    "    \"congress_count\":[]\n",
    "}\n",
    "size=data.read_data().shape[0]\n",
    "for j in range(size):\n",
    "    count=0\n",
    "    #data.clean_dataset()['Article'][i]\n",
    "    article=data.clean_dataset()['Article'][j]\n",
    "    tokens=word_tokenize(article)\n",
    "    stopwordsInArticle,stopwordsNotInArticle=remove_stop_words(tokens)\n",
    "    stopwordsNotInArticle=lemmatise(stopwordsNotInArticle)\n",
    "    for i in range(len(stopwordsNotInArticle)):\n",
    "        length=len(stopwordsNotInArticle)\n",
    "        result=congress(stopwordsNotInArticle[i])\n",
    "        if(result==True):\n",
    "            count=count+1\n",
    "    count_congress['article'].append(j)\n",
    "    count_congress['congress_count'].append(count/length)\n",
    "print(count_congress)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_congress = pd.DataFrame(count_congress)\n",
    "df_congress['congress_count']=df_congress['congress_count']*100\n",
    "df_congress.to_csv('congress_count.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article</th>\n",
       "      <th>bjp_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.448980</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5.027933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3.269755</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3.980100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2.823529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>220</td>\n",
       "      <td>220</td>\n",
       "      <td>3.197158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>221</td>\n",
       "      <td>221</td>\n",
       "      <td>1.865672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>222</td>\n",
       "      <td>222</td>\n",
       "      <td>2.657005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>223</td>\n",
       "      <td>223</td>\n",
       "      <td>0.469484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>224</td>\n",
       "      <td>224</td>\n",
       "      <td>0.172117</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>225 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     article  bjp_count\n",
       "0          0   2.448980\n",
       "1          1   5.027933\n",
       "2          2   3.269755\n",
       "3          3   3.980100\n",
       "4          4   2.823529\n",
       "..       ...        ...\n",
       "220      220   3.197158\n",
       "221      221   1.865672\n",
       "222      222   2.657005\n",
       "223      223   0.469484\n",
       "224      224   0.172117\n",
       "\n",
       "[225 rows x 2 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_bjp = pd.DataFrame(count_bjp)\n",
    "df_bjp['bjp_count']=df_bjp['bjp_count']*100\n",
    "df_bjp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_bjp.to_csv('bjp_count.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "def bjp(string):\n",
    "    \n",
    "    file=open(\"data/bias-lexicon/congress-corpus.txt\",\"r\")\n",
    "\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        for word in words:\n",
    "            if word == string:\n",
    "                #print(\"found\")\n",
    "                #print(words[0] , words[2])\n",
    "               \n",
    "                return True\n",
    "    return False\n",
    "    file.close()\n",
    "count=0\n",
    "for i in range(len(stopwordsNotInArticle)):\n",
    "    result=bjp(stopwordsNotInArticle[i])\n",
    "    if(result==True):\n",
    "        count=count+1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(df)):\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def strongSubjectivtiy(string):\n",
    "\n",
    "    file=open(\"data/bias-lexicon/subjandpolar.txt\",\"r\")\n",
    "\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        for word in words:\n",
    "            if word == string:\n",
    "                #print(\"found\")\n",
    "                #print(words[0] , words[2])\n",
    "                if(words[0]=='type=strongsubj'):\n",
    "                    return True\n",
    "    return False\n",
    "    file.close()\n",
    "\n",
    "string=\"abuse\"\n",
    "strongSubjectivtiy(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "def strongSubjectivityContext(article,word,index):\n",
    "    length=len(article)-1\n",
    "    flag=False\n",
    "    if index==0:\n",
    "        if(strongSubjectivtiy(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(strongSubjectivtiy(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==1:\n",
    "        if(strongSubjectivtiy(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(strongSubjectivtiy(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(strongSubjectivtiy(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length:\n",
    "        if(strongSubjectivtiy(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(strongSubjectivtiy(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length-1:\n",
    "        if(strongSubjectivtiy(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(strongSubjectivtiy(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(strongSubjectivtiy(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index>1 and index<length-1:\n",
    "        if(strongSubjectivtiy(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(strongSubjectivtiy(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(strongSubjectivtiy(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(strongSubjectivtiy(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    return flag\n",
    "\n",
    "print(strongSubjectivityContext(stopwordsNotInArticle,stopwordsNotInArticle[4],4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def weakSubjectivtiy(string):\n",
    "\n",
    "    file=open(\"data/bias-lexicon/subjandpolar.txt\",\"r\")\n",
    "\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        for word in words:\n",
    "            if word == string:\n",
    "                #print(\"found\")\n",
    "                #print(words[0] , words[2])\n",
    "                if(words[0]=='type=weaksubj'):\n",
    "                    return True\n",
    "    return False\n",
    "    file.close()\n",
    "\n",
    "string=\"abate\"\n",
    "weakSubjectivtiy(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "def weakSubjectivityContext(article,word,index):\n",
    "    length=len(article)-1\n",
    "    flag=False\n",
    "    if index==0:\n",
    "        if(weakSubjectivtiy(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(weakSubjectivtiy(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==1:\n",
    "        if(weakSubjectivtiy(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(weakSubjectivtiy(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(weakSubjectivtiy(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length:\n",
    "        if(weakSubjectivtiy(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(weakSubjectivtiy(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length-1:\n",
    "        if(weakSubjectivtiy(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(weakSubjectivtiy(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(weakSubjectivtiy(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index>1 and index<length-1:\n",
    "        if(weakSubjectivtiy(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(weakSubjectivtiy(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(weakSubjectivtiy(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(weakSubjectivtiy(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    return flag\n",
    "\n",
    "print(weakSubjectivityContext(stopwordsNotInArticle,stopwordsNotInArticle[22],22))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def biasLexion(string):\n",
    "\n",
    "    file=open(\"data/bias-lexicon/bias-lexicon.txt\",\"r\")\n",
    "\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        for word in words:\n",
    "            if word == string:\n",
    "                #print(word)\n",
    "                #print(words[0] , words[2])\n",
    "                return True\n",
    "    return False\n",
    "    file.close()\n",
    "\n",
    "string=\"west\"\n",
    "biasLexion(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "22\n",
      "490\n"
     ]
    }
   ],
   "source": [
    "def positivewords(string):\n",
    "    \n",
    "    file=open(\"data/bias-lexicon/positive-words.txt\",\"r\")\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        positive = False\n",
    "        if words[0] == string:\n",
    "            positive = True\n",
    "            return positive\n",
    "        \n",
    "    return positive\n",
    "    file.close()\n",
    "\n",
    "string=\"accurately\"\n",
    "print(positivewords(string))\n",
    "\n",
    "\n",
    "count=[]\n",
    "for i in range(len(stopwordsNotInArticle)):\n",
    "    count.append(positivewords(stopwordsNotInArticle[i]))\n",
    "#print(count)\n",
    "true=[]\n",
    "for i in range(len(count)):\n",
    "    if count[i]==True:\n",
    "        true.append(count[i])\n",
    "true\n",
    "print(len(true))\n",
    "print(len(stopwordsNotInArticle))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "def positiveWordContext(article,word,index):\n",
    "    length=len(article)-1\n",
    "    flag=False\n",
    "    if index==0:\n",
    "        if(positivewords(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(positivewords(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==1:\n",
    "        if(positivewords(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(positivewords(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(positivewords(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length:\n",
    "        if(positivewords(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(positivewords(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length-1:\n",
    "        if(positivewords(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(positivewords(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(positivewords(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index>1 and index<length-1:\n",
    "        if(positivewords(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(positivewords(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(positivewords(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(positivewords(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    return flag\n",
    "\n",
    "print(positiveWordContext(stopwordsNotInArticle,stopwordsNotInArticle[22],22))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n",
      "21\n",
      "490\n"
     ]
    }
   ],
   "source": [
    "def negativewords(string):\n",
    "    \n",
    "    file=open(\"data/bias-lexicon/negative-words.txt\",\"r\")\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        negative = False\n",
    "        if words[0] == string:\n",
    "            negative = True\n",
    "            return negative\n",
    "        \n",
    "    return negative\n",
    "    file.close()\n",
    "string=\"good\"\n",
    "print(negativewords(string))\n",
    "\n",
    "count=[]\n",
    "for i in range(len(stopwordsNotInArticle)):\n",
    "    count.append(negativewords(stopwordsNotInArticle[i]))\n",
    "#print(count)\n",
    "true=[]\n",
    "for i in range(len(count)):\n",
    "    if count[i]==True:\n",
    "        true.append(count[i])\n",
    "true\n",
    "print(len(true))\n",
    "print(len(stopwordsNotInArticle))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "def negativeWordContext(article,word,index):\n",
    "    length=len(article)-1\n",
    "    flag=False\n",
    "    if index==0:\n",
    "        if(negativewords(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(negativewords(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==1:\n",
    "        if(negativewords(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(negativewords(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(negativewords(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length:\n",
    "        if(negativewords(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(negativewords(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length-1:\n",
    "        if(negativewords(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(negativewords(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(negativewords(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index>1 and index<length-1:\n",
    "        if(negativewords(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(negativewords(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(negativewords(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(negativewords(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    return flag\n",
    "\n",
    "print(negativeWordContext(stopwordsNotInArticle,stopwordsNotInArticle[17],17))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "3\n",
      "490\n"
     ]
    }
   ],
   "source": [
    "def factives_hooper(string):\n",
    "    \n",
    "    file=open(\"data/bias-lexicon/factives_hooper1975.txt\",\"r\")\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        factives = False\n",
    "        if words[0] == string:\n",
    "            factives = True\n",
    "            return factives\n",
    "        \n",
    "    return factives\n",
    "    file.close()\n",
    "string=\"care\"\n",
    "print(factives_hooper(string))\n",
    "\n",
    "\n",
    "count=[]\n",
    "for i in range(len(stopwordsNotInArticle)):\n",
    "    count.append(factives_hooper(stopwordsNotInArticle[i]))\n",
    "#print(count)\n",
    "true=[]\n",
    "for i in range(len(count)):\n",
    "    if count[i]==True:\n",
    "        true.append(count[i])\n",
    "true\n",
    "print(len(true))\n",
    "print(len(stopwordsNotInArticle))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "def factives_hooper_context(article,word,index):\n",
    "    length=len(article)-1\n",
    "    flag=False\n",
    "    if index==0:\n",
    "        if(factives_hooper(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(factives_hooper(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==1:\n",
    "        if(factives_hooper(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(factives_hooper(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(factives_hooper(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length:\n",
    "        if(factives_hooper(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(factives_hooper(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length-1:\n",
    "        if(factives_hooper(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(factives_hooper(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(factives_hooper(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index>1 and index<length-1:\n",
    "        if(factives_hooper(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(factives_hooper(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(factives_hooper(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(factives_hooper(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    return flag\n",
    "\n",
    "print(factives_hooper_context(stopwordsNotInArticle,stopwordsNotInArticle[17],17))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "9\n",
      "490\n"
     ]
    }
   ],
   "source": [
    "def hedges(string):\n",
    "    file=open(\"data/bias-lexicon/hedges_hyland2005.txt\",\"r\")\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        hedge = False\n",
    "        if words[0] == string:\n",
    "            hedge = True\n",
    "            return hedge\n",
    "    return hedge\n",
    "    file.close()\n",
    "string=\"largely\"\n",
    "print(hedges(string))\n",
    "\n",
    "\n",
    "count=[]\n",
    "for i in range(len(stopwordsNotInArticle)):\n",
    "    count.append(hedges(stopwordsNotInArticle[i]))\n",
    "#print(count)\n",
    "true=[]\n",
    "for i in range(len(count)):\n",
    "    if count[i]==True:\n",
    "        true.append(count[i])\n",
    "true\n",
    "print(len(true))\n",
    "print(len(stopwordsNotInArticle))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "def hedges_context(article,word,index):\n",
    "    length=len(article)-1\n",
    "    flag=False\n",
    "    if index==0:\n",
    "        if(hedges(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(hedges(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==1:\n",
    "        if(hedges(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(hedges(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(hedges(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length:\n",
    "        if(hedges(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(hedges(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length-1:\n",
    "        if(hedges(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(hedges(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(hedges(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index>1 and index<length-1:\n",
    "        if(hedges(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(hedges(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(hedges(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(hedges(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    return flag\n",
    "\n",
    "print(hedges_context(stopwordsNotInArticle,stopwordsNotInArticle[337],337))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "22\n",
      "490\n"
     ]
    }
   ],
   "source": [
    "def assertive_hooper(string):\n",
    "    file=open(\"data/bias-lexicon/assertives_hooper1975.txt\",\"r\")\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        assertives = False\n",
    "        if words[0] == string:\n",
    "            assertives = True\n",
    "            return assertives\n",
    "    return assertives\n",
    "    file.close()\n",
    "string=\"maintain\"\n",
    "print(assertive_hooper(string))\n",
    "\n",
    "\n",
    "count=[]\n",
    "for i in range(len(stopwordsNotInArticle)):\n",
    "    count.append(assertive_hooper(stopwordsNotInArticle[i]))\n",
    "#print(count)\n",
    "true=[]\n",
    "for i in range(len(count)):\n",
    "    if count[i]==True:\n",
    "        true.append(count[i])\n",
    "true\n",
    "print(len(true))\n",
    "print(len(stopwordsNotInArticle))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "False\n"
     ]
    }
   ],
   "source": [
    "def assertive_context(article,word,index):\n",
    "    length=len(article)-1\n",
    "    flag=False\n",
    "    if index==0:\n",
    "        if(assertive_hooper(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(assertive_hooper(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==1:\n",
    "        if(assertive_hooper(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(assertive_hooper(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(assertive_hooper(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length:\n",
    "        if(assertive_hooper(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(assertive_hooper(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length-1:\n",
    "        if(assertive_hooper(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(assertive_hooper(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(assertive_hooper(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index>1 and index<length-1:\n",
    "        if(assertive_hooper(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(assertive_hooper(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(assertive_hooper(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(assertive_hooper(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    return flag\n",
    "\n",
    "print(assertive_context(stopwordsNotInArticle,stopwordsNotInArticle[0],0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "36\n",
      "490\n"
     ]
    }
   ],
   "source": [
    "def report_verb(string):\n",
    "    file=open(\"data/bias-lexicon/report_verbs.txt\",\"r\")\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        report = False\n",
    "        if words[0] == string:\n",
    "            report = True\n",
    "            return report\n",
    "    return report\n",
    "    file.close()\n",
    "string=\"caution\"\n",
    "print(report_verb(string))\n",
    "\n",
    "count=[]\n",
    "for i in range(len(stopwordsNotInArticle)):\n",
    "    count.append(report_verb(stopwordsNotInArticle[i]))\n",
    "#print(count)\n",
    "true=[]\n",
    "for i in range(len(count)):\n",
    "    if count[i]==True:\n",
    "        true.append(count[i])\n",
    "true\n",
    "print(len(true))\n",
    "print(len(stopwordsNotInArticle))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "def report_verb_context(article,word,index):\n",
    "    length=len(article)-1\n",
    "    flag=False\n",
    "    if index==0:\n",
    "        if(report_verb(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(report_verb(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==1:\n",
    "        if(report_verb(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(report_verb(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(report_verb(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length:\n",
    "        if(report_verb(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(report_verb(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length-1:\n",
    "        if(report_verb(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(report_verb(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(report_verb(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index>1 and index<length-1:\n",
    "        if(report_verb(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(report_verb(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(report_verb(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(report_verb(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    return flag\n",
    "\n",
    "print(report_verb_context(stopwordsNotInArticle,stopwordsNotInArticle[43],43))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n",
      "3\n",
      "490\n"
     ]
    }
   ],
   "source": [
    "def implicative_verb(string):\n",
    "    file=open(\"data/bias-lexicon/implicatives_karttunen1971.txt\",\"r\")\n",
    "    for line in file:\n",
    "        words=line.split()\n",
    "        implicative = False\n",
    "        if words[0] == string:\n",
    "            implicative = True\n",
    "            return implicative\n",
    "    return implicative\n",
    "    file.close()\n",
    "string=\"bother\"\n",
    "print(implicative_verb(string))\n",
    "\n",
    "count=[]\n",
    "for i in range(len(stopwordsNotInArticle)):\n",
    "    count.append(implicative_verb(stopwordsNotInArticle[i]))\n",
    "#print(count)\n",
    "true=[]\n",
    "for i in range(len(count)):\n",
    "    if count[i]==True:\n",
    "        true.append(count[i])\n",
    "true\n",
    "print(len(true))\n",
    "print(len(stopwordsNotInArticle))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "def implicative_verb_context(article,word,index):\n",
    "    length=len(article)-1\n",
    "    flag=False\n",
    "    if index==0:\n",
    "        if(implicative_verb(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(implicative_verb(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==1:\n",
    "        if(implicative_verb(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(implicative_verb(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(implicative_verb(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length:\n",
    "        if(implicative_verb(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(implicative_verb(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index==length-1:\n",
    "        if(implicative_verb(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(implicative_verb(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(implicative_verb(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    if index>1 and index<length-1:\n",
    "        if(implicative_verb(article[index-2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(implicative_verb(article[index-1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(implicative_verb(article[index+1])):\n",
    "            flag=True\n",
    "            return flag\n",
    "        if(implicative_verb(article[index+2])):\n",
    "            flag=True\n",
    "            return flag\n",
    "    return flag\n",
    "\n",
    "print(implicative_verb_context(stopwordsNotInArticle,stopwordsNotInArticle[443],443))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "358\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'NN'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def pos(string):\n",
    "    tagged = nltk.pos_tag([string])\n",
    "    #print(tagged[0][1])\n",
    "    return tagged[0][1]\n",
    "    \n",
    "def posNeg1(article,index):\n",
    "    if index==0:\n",
    "        return 'none'\n",
    "    else:\n",
    "        string=article[index-1]\n",
    "        tagged = nltk.pos_tag([string])\n",
    "        return tagged[0][1]\n",
    "    \n",
    "def posNeg2(article,index):\n",
    "    if index==0 or index==1:\n",
    "        return 'none'\n",
    "    else:\n",
    "        string=article[index-2]\n",
    "        tagged = nltk.pos_tag([string])\n",
    "        return tagged[0][1]\n",
    "def pos1(article,index):\n",
    "    length=len(article)-1\n",
    "    if index==length:\n",
    "        return 'none'\n",
    "    else:\n",
    "        string=article[index+1]\n",
    "        tagged = nltk.pos_tag([string])\n",
    "        return tagged[0][1]\n",
    "def pos2(article,index):\n",
    "    length=len(article)-1\n",
    "    if index==length or index==length-1:\n",
    "        return 'none'\n",
    "    else:\n",
    "        string=article[index+1]\n",
    "        tagged = nltk.pos_tag([string])\n",
    "        return tagged[0][1]\n",
    "print(len(stopwordsNotInArticle))\n",
    "posNeg2(stopwordsNotInArticle,198)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'word': [],\n",
       " 'POS': [],\n",
       " 'POSNeg1': [],\n",
       " 'POSNeg2': [],\n",
       " 'POS1': [],\n",
       " 'POS2': [],\n",
       " 'Hedge': [],\n",
       " 'HedgeContext': [],\n",
       " 'FativeVerb': [],\n",
       " 'FactiveVerbContext': [],\n",
       " 'AssertiveVerb': [],\n",
       " 'AssertiveVerbContext': [],\n",
       " 'ImplicativeVerb': [],\n",
       " 'ImplicativeVerbContext': [],\n",
       " 'ReportVerb': [],\n",
       " 'ReportVerbContext': [],\n",
       " 'StrongSub': [],\n",
       " 'StrongSubContext': [],\n",
       " 'WeakSub': [],\n",
       " 'WeakSubContext': [],\n",
       " 'PositiveWord': [],\n",
       " 'PositiveWordContext': [],\n",
       " 'NegativeWord': [],\n",
       " 'NegativeWordContext': [],\n",
       " 'BiasLexicon': []}"
      ]
     },
     "execution_count": 245,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#List of Lingusitic features to be considered while analysing a token\n",
    "\n",
    "features = {\n",
    "        'word':[],\n",
    "        'POS':[],\n",
    "        'POSNeg1':[],\n",
    "        'POSNeg2':[],\n",
    "        'POS1':[],\n",
    "        'POS2':[],\n",
    "        'Hedge':[],\n",
    "        'HedgeContext':[],\n",
    "        'FativeVerb':[],\n",
    "        'FactiveVerbContext':[],\n",
    "        'AssertiveVerb':[],\n",
    "        'AssertiveVerbContext':[],\n",
    "        'ImplicativeVerb':[],\n",
    "        'ImplicativeVerbContext':[],\n",
    "        'ReportVerb':[],\n",
    "        'ReportVerbContext':[],\n",
    "        'StrongSub':[],\n",
    "        'StrongSubContext':[],\n",
    "        'WeakSub':[],\n",
    "        'WeakSubContext':[],\n",
    "        'PositiveWord':[],\n",
    "        'PositiveWordContext':[],\n",
    "        'NegativeWord':[],\n",
    "        'NegativeWordContext':[],\n",
    "        'BiasLexicon':[]\n",
    "}\n",
    "\n",
    "features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "78\n"
     ]
    }
   ],
   "source": [
    "size=data.read_data().shape[0]\n",
    "print(size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:19: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:21: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Hedge': [1.8367346938775513, 3.910614525139665, 2.452316076294278, 4.643449419568822, 1.411764705882353, 1.8975332068311195, 0.6651884700665188, 2.4752475247524752, 6.467661691542288, 1.7391304347826086, 2.092050209205021, 2.460456942003515, 0.26420079260237783, 3.0534351145038165, 1.0460251046025104, 2.734375, 1.5625, 3.2653061224489797, 2.3255813953488373, 1.6597510373443984, 0.41841004184100417, 0.8368200836820083, 1.9157088122605364, 1.9607843137254901, 3.4934497816593884, 3.3264033264033266, 4.365079365079365, 1.834862385321101, 2.4875621890547266, 3.571428571428571, 2.6548672566371683, 1.9305019305019304, 2.0161290322580645, 4.444444444444445, 2.3255813953488373, 1.876675603217158, 3.4482758620689653, 1.9607843137254901, 2.2222222222222223, 2.5936599423631126, 0.4, 1.7543859649122806, 0.6, 3.546099290780142, 1.8779342723004695, 0.0, 1.2237762237762237, 1.312910284463895, 2.2821576763485476, 0.6825938566552902, 2.888888888888889, 1.7994858611825193, 2.2883295194508007, 2.2813688212927756, 0.8695652173913043, 1.8675721561969438, 1.7518248175182483, 2.5974025974025974, 2.8000000000000003, 2.28310502283105, 3.7735849056603774, 3.3816425120772946, 2.3346303501945527, 3.7209302325581395, 1.1142061281337048, 2.3923444976076556, 1.8382352941176472, 4.195804195804196, 1.5544041450777202, 2.6143790849673203, 1.8604651162790697, 1.9498607242339834, 1.1152416356877324, 2.3923444976076556, 0.46511627906976744, 2.1645021645021645, 1.6666666666666667, 1.3311148086522462], 'HedgeContext': [6.938775510204081, 14.24581005586592, 9.809264305177113, 17.744610281923716, 4.705882352941177, 7.2106261859582546, 2.6607538802660753, 9.900990099009901, 23.383084577114428, 6.086956521739131, 8.368200836820083, 9.666080843585236, 1.0568031704095113, 11.450381679389313, 4.184100418410042, 10.546875, 5.208333333333334, 11.428571428571429, 9.30232558139535, 6.639004149377594, 1.6736401673640167, 3.3472803347280333, 6.896551724137931, 7.8431372549019605, 13.318777292576419, 12.266112266112266, 16.071428571428573, 6.422018348623854, 9.950248756218906, 14.285714285714285, 10.619469026548673, 7.335907335907336, 7.258064516129033, 17.77777777777778, 8.637873754152823, 6.970509383378016, 13.30049261083744, 7.8431372549019605, 8.88888888888889, 9.798270893371757, 1.6, 7.017543859649122, 2.4, 13.47517730496454, 7.511737089201878, 0.0, 4.895104895104895, 5.25164113785558, 8.921161825726141, 2.7303754266211606, 10.888888888888888, 6.169665809768637, 9.153318077803203, 8.745247148288973, 3.4782608695652173, 7.470288624787775, 6.569343065693431, 9.956709956709958, 11.200000000000001, 9.1324200913242, 14.150943396226415, 13.526570048309178, 9.33852140077821, 14.883720930232558, 4.178272980501393, 9.569377990430622, 7.352941176470589, 16.433566433566433, 5.699481865284974, 10.457516339869281, 7.441860465116279, 7.7994428969359335, 4.4609665427509295, 9.569377990430622, 1.8604651162790697, 8.225108225108226, 6.666666666666667, 4.9916805324459235], 'FativeVerb': [0.6122448979591837, 0.27932960893854747, 0.544959128065395, 1.8242122719734661, 0.2352941176470588, 3.225806451612903, 0.6651884700665188, 0.49504950495049505, 0.4975124378109453, 0.43478260869565216, 0.8368200836820083, 0.8787346221441126, 0.6605019815059445, 0.38167938931297707, 0.41841004184100417, 0.78125, 0.5208333333333333, 0.0, 0.0, 0.8298755186721992, 0.41841004184100417, 0.41841004184100417, 1.1494252873563218, 0.9803921568627451, 1.0917030567685588, 1.6632016632016633, 1.1904761904761905, 1.834862385321101, 1.4925373134328357, 1.5873015873015872, 1.7699115044247788, 3.474903474903475, 0.8064516129032258, 5.185185185185185, 0.33222591362126247, 0.5361930294906166, 0.9852216748768473, 0.39215686274509803, 1.6666666666666667, 3.170028818443804, 1.2, 0.29239766081871343, 0.4, 0.0, 0.4694835680751174, 1.3274336283185841, 1.3986013986013985, 0.87527352297593, 0.8298755186721992, 0.0, 0.8888888888888888, 1.2853470437017995, 0.9153318077803204, 0.38022813688212925, 0.0, 1.1884550084889642, 1.897810218978102, 1.2987012987012987, 0.0, 0.91324200913242, 0.4716981132075472, 1.4492753623188406, 0.7782101167315175, 0.46511627906976744, 0.2785515320334262, 1.4354066985645932, 0.7352941176470588, 0.5244755244755245, 1.0362694300518136, 2.6143790849673203, 1.3953488372093024, 1.392757660167131, 1.486988847583643, 0.9569377990430622, 0.9302325581395349, 0.8658008658008658, 2.7777777777777777, 1.9966722129783694], 'FactiveVerbContext': [2.4489795918367347, 1.1173184357541899, 1.9073569482288828, 7.131011608623548, 0.9411764705882352, 12.333965844402277, 2.6607538802660753, 1.9801980198019802, 1.9900497512437811, 1.7391304347826086, 3.3472803347280333, 3.5149384885764503, 2.642007926023778, 1.5267175572519083, 1.6736401673640167, 3.125, 2.083333333333333, 0.0, 0.0, 3.319502074688797, 1.6736401673640167, 1.6736401673640167, 4.597701149425287, 3.9215686274509802, 4.366812227074235, 6.652806652806653, 4.761904761904762, 7.339449541284404, 5.472636815920398, 5.952380952380952, 7.079646017699115, 12.355212355212355, 3.225806451612903, 18.51851851851852, 1.3289036544850499, 2.1447721179624666, 3.9408866995073892, 1.5686274509803921, 6.666666666666667, 12.103746397694524, 4.0, 1.1695906432748537, 1.6, 0.0, 1.8779342723004695, 5.3097345132743365, 5.594405594405594, 3.50109409190372, 3.319502074688797, 0.0, 3.111111111111111, 4.370179948586118, 3.6613272311212817, 1.520912547528517, 0.0, 4.753820033955857, 7.445255474452554, 5.194805194805195, 0.0, 3.1963470319634704, 1.8867924528301887, 5.797101449275362, 2.3346303501945527, 1.8604651162790697, 1.1142061281337048, 5.741626794258373, 2.941176470588235, 2.097902097902098, 4.145077720207254, 8.49673202614379, 5.5813953488372094, 5.571030640668524, 5.204460966542751, 3.827751196172249, 3.7209302325581395, 3.463203463203463, 11.11111111111111, 7.653910149750416], 'AssertiveVerb': [4.489795918367347, 3.072625698324022, 4.35967302452316, 5.970149253731343, 3.294117647058824, 2.6565464895635675, 4.878048780487805, 2.4752475247524752, 5.970149253731343, 1.3043478260869565, 2.510460251046025, 2.987697715289982, 2.2457067371202113, 3.435114503816794, 2.510460251046025, 2.5390625, 1.0416666666666665, 2.0408163265306123, 1.937984496124031, 1.2448132780082988, 1.2552301255230125, 1.2552301255230125, 4.21455938697318, 3.1862745098039214, 4.148471615720524, 1.4553014553014554, 2.976190476190476, 2.7522935779816518, 4.477611940298507, 5.555555555555555, 6.1946902654867255, 5.405405405405405, 1.6129032258064515, 0.7407407407407408, 1.3289036544850499, 7.774798927613941, 3.9408866995073892, 5.88235294117647, 3.3333333333333335, 7.492795389048991, 2.8000000000000003, 2.631578947368421, 1.4000000000000001, 2.127659574468085, 4.694835680751173, 0.8849557522123894, 2.272727272727273, 0.6564551422319475, 1.8672199170124482, 2.7303754266211606, 3.5555555555555554, 2.570694087403599, 3.203661327231121, 3.041825095057034, 3.4782608695652173, 2.037351443123939, 2.9197080291970803, 3.0303030303030303, 2.0, 2.73972602739726, 2.358490566037736, 2.4154589371980677, 1.556420233463035, 1.8604651162790697, 1.392757660167131, 3.349282296650718, 1.8382352941176472, 2.272727272727273, 3.8860103626943006, 3.2679738562091507, 1.8604651162790697, 3.6211699164345403, 4.089219330855019, 2.3923444976076556, 2.3255813953488373, 3.896103896103896, 5.0, 5.158069883527454], 'AssertiveVerbContext': [17.346938775510203, 12.290502793296088, 17.166212534059948, 22.553897180762853, 11.76470588235294, 10.056925996204933, 17.96008869179601, 9.405940594059405, 23.88059701492537, 5.217391304347826, 9.623430962343097, 10.896309314586995, 8.45442536327609, 13.358778625954198, 9.414225941422593, 9.9609375, 4.166666666666666, 7.755102040816326, 7.751937984496124, 4.979253112033195, 5.02092050209205, 4.184100418410042, 14.17624521072797, 11.76470588235294, 15.72052401746725, 5.8212058212058215, 11.507936507936508, 11.009174311926607, 16.91542288557214, 21.428571428571427, 23.008849557522122, 20.077220077220076, 6.048387096774194, 1.4814814814814816, 5.3156146179401995, 28.150134048257375, 15.270935960591133, 21.176470588235293, 13.333333333333334, 25.360230547550433, 11.200000000000001, 10.526315789473683, 5.6000000000000005, 8.51063829787234, 17.84037558685446, 3.5398230088495577, 8.916083916083917, 2.62582056892779, 7.468879668049793, 10.580204778156997, 14.222222222222221, 9.768637532133676, 12.356979405034325, 11.02661596958175, 12.173913043478262, 7.979626485568761, 11.386861313868613, 12.121212121212121, 8.0, 9.817351598173515, 9.433962264150944, 8.695652173913043, 6.22568093385214, 7.441860465116279, 5.571030640668524, 12.440191387559809, 6.985294117647059, 8.741258741258742, 15.284974093264248, 13.071895424836603, 7.441860465116279, 13.370473537604457, 16.356877323420075, 9.569377990430622, 9.30232558139535, 15.151515151515152, 18.333333333333332, 19.966722129783694], 'ImplicativeVerb': [0.6122448979591837, 1.1173184357541899, 0.8174386920980926, 0.4975124378109453, 0.9411764705882352, 0.9487666034155597, 0.6651884700665188, 1.4851485148514851, 1.9900497512437811, 0.0, 2.092050209205021, 1.2302284710017575, 1.453104359313078, 0.38167938931297707, 0.8368200836820083, 1.5625, 3.125, 0.40816326530612246, 1.1627906976744187, 1.2448132780082988, 0.0, 0.8368200836820083, 1.1494252873563218, 0.9803921568627451, 0.8733624454148471, 2.2869022869022873, 0.3968253968253968, 0.9174311926605505, 0.9950248756218906, 1.1904761904761905, 1.7699115044247788, 1.1583011583011582, 0.8064516129032258, 0.0, 0.6644518272425249, 1.3404825737265416, 1.477832512315271, 1.9607843137254901, 3.888888888888889, 0.2881844380403458, 0.8, 0.29239766081871343, 0.8, 0.7092198581560284, 0.4694835680751174, 0.4424778761061947, 1.2237762237762237, 1.9693654266958425, 0.8298755186721992, 0.6825938566552902, 0.6666666666666667, 0.2570694087403599, 0.6864988558352403, 1.520912547528517, 0.8695652173913043, 3.0560271646859083, 2.335766423357664, 1.7316017316017316, 0.0, 2.28310502283105, 1.4150943396226416, 0.4830917874396135, 0.7782101167315175, 1.8604651162790697, 0.2785515320334262, 0.9569377990430622, 1.4705882352941175, 2.797202797202797, 2.072538860103627, 2.6143790849673203, 0.46511627906976744, 0.2785515320334262, 0.7434944237918215, 0.0, 0.9302325581395349, 0.0, 2.2222222222222223, 0.6655574043261231], 'ImplicativeVerbContext': [2.4489795918367347, 4.4692737430167595, 3.2697547683923704, 1.8242122719734661, 3.7647058823529407, 3.795066413662239, 2.6607538802660753, 5.9405940594059405, 7.960199004975125, 0.0, 8.368200836820083, 4.92091388400703, 5.680317040951123, 1.5267175572519083, 3.3472803347280333, 6.25, 11.979166666666668, 1.6326530612244898, 4.651162790697675, 4.979253112033195, 0.0, 3.3472803347280333, 4.597701149425287, 3.9215686274509802, 3.2751091703056767, 9.147609147609149, 1.5873015873015872, 3.669724770642202, 3.9800995024875623, 4.761904761904762, 7.079646017699115, 4.633204633204633, 3.225806451612903, 0.0, 2.6578073089700998, 5.361930294906166, 4.926108374384237, 7.8431372549019605, 15.555555555555555, 1.1527377521613833, 2.8000000000000003, 1.1695906432748537, 3.2, 2.8368794326241136, 1.8779342723004695, 1.7699115044247788, 4.545454545454546, 7.2210065645514225, 3.319502074688797, 2.7303754266211606, 2.2222222222222223, 1.0282776349614395, 2.745995423340961, 5.7034220532319395, 3.4782608695652173, 11.544991511035652, 8.75912408759124, 6.0606060606060606, 0.0, 9.1324200913242, 5.660377358490567, 1.932367149758454, 3.11284046692607, 7.441860465116279, 1.1142061281337048, 3.827751196172249, 5.514705882352941, 11.013986013986015, 7.512953367875648, 10.457516339869281, 1.8604651162790697, 1.1142061281337048, 2.973977695167286, 0.0, 3.7209302325581395, 0.0, 8.88888888888889, 2.6622296173044924], 'ReportVerb': [7.346938775510205, 6.145251396648044, 4.632152588555858, 7.960199004975125, 6.8235294117647065, 10.436432637571158, 7.317073170731707, 6.9306930693069315, 9.950248756218906, 8.695652173913043, 7.531380753138076, 5.623901581722319, 4.359313077939234, 5.343511450381679, 8.786610878661087, 7.421875, 5.729166666666666, 5.3061224489795915, 3.875968992248062, 4.979253112033195, 8.368200836820083, 9.623430962343097, 6.130268199233716, 8.57843137254902, 6.550218340611353, 4.781704781704782, 6.349206349206349, 9.174311926605505, 6.965174129353234, 12.698412698412698, 12.389380530973451, 13.127413127413126, 5.241935483870968, 6.666666666666667, 3.6544850498338874, 10.187667560321715, 3.9408866995073892, 9.803921568627452, 8.333333333333332, 13.256484149855908, 6.4, 7.894736842105263, 4.6, 2.8368794326241136, 10.328638497652582, 3.0973451327433628, 5.769230769230769, 3.282275711159737, 4.149377593360995, 5.802047781569966, 6.444444444444445, 9.254498714652955, 5.263157894736842, 7.224334600760455, 4.3478260869565215, 2.5466893039049237, 4.817518248175182, 6.0606060606060606, 3.5999999999999996, 5.93607305936073, 4.716981132075472, 5.797101449275362, 3.501945525291829, 5.116279069767442, 5.2924791086350975, 7.655502392344498, 5.514705882352941, 3.8461538461538463, 7.253886010362693, 1.9607843137254901, 6.511627906976744, 5.013927576601671, 8.550185873605948, 5.263157894736842, 6.511627906976744, 9.090909090909092, 11.11111111111111, 10.98169717138103], 'ReportVerbContext': [27.55102040816326, 22.62569832402235, 18.256130790190735, 28.192371475953564, 24.0, 36.81214421252372, 26.164079822616408, 24.752475247524753, 35.32338308457712, 32.17391304347826, 25.523012552301257, 19.85940246045694, 15.587846763540291, 19.84732824427481, 32.21757322175732, 26.5625, 19.270833333333336, 19.591836734693878, 14.728682170542637, 18.672199170124482, 28.451882845188287, 34.30962343096235, 21.455938697318008, 29.411764705882355, 22.489082969432314, 19.12681912681913, 23.61111111111111, 26.605504587155966, 27.363184079601986, 46.42857142857143, 40.26548672566372, 45.559845559845556, 19.35483870967742, 22.22222222222222, 14.285714285714285, 34.316353887399465, 14.77832512315271, 36.86274509803922, 32.77777777777778, 38.90489913544668, 22.400000000000002, 27.485380116959064, 16.0, 9.929078014184398, 36.15023474178404, 12.389380530973451, 21.853146853146853, 12.472647702407002, 15.560165975103734, 20.13651877133106, 23.77777777777778, 32.13367609254499, 20.137299771167047, 25.09505703422053, 17.391304347826086, 9.168081494057725, 18.54014598540146, 21.645021645021643, 14.000000000000002, 22.146118721461185, 16.50943396226415, 21.73913043478261, 14.007782101167315, 19.53488372093023, 18.94150417827298, 29.1866028708134, 20.588235294117645, 14.51048951048951, 26.165803108808287, 7.18954248366013, 21.86046511627907, 18.94150417827298, 31.226765799256505, 20.574162679425836, 23.72093023255814, 34.63203463203463, 37.77777777777778, 37.770382695507486], 'StrongSub': [8.979591836734693, 8.659217877094973, 13.079019073569482, 10.281923714759536, 8.0, 8.538899430740038, 5.764966740576496, 9.900990099009901, 15.92039800995025, 16.956521739130434, 17.573221757322173, 7.381370826010544, 5.019815059445178, 15.648854961832063, 7.112970711297072, 14.0625, 11.458333333333332, 9.795918367346939, 4.651162790697675, 6.224066390041494, 17.99163179916318, 15.899581589958158, 13.409961685823754, 11.519607843137255, 17.248908296943235, 9.355509355509357, 13.095238095238097, 15.59633027522936, 11.940298507462686, 7.142857142857142, 7.079646017699115, 7.335907335907336, 17.741935483870968, 14.814814814814813, 6.976744186046512, 8.310991957104557, 13.793103448275861, 9.411764705882353, 13.88888888888889, 7.204610951008646, 14.000000000000002, 12.280701754385964, 11.799999999999999, 13.47517730496454, 13.615023474178404, 14.601769911504425, 12.587412587412588, 12.25382932166302, 6.016597510373444, 9.556313993174061, 10.222222222222223, 11.053984575835475, 16.475972540045767, 11.02661596958175, 12.173913043478262, 11.205432937181664, 9.48905109489051, 11.255411255411255, 10.0, 10.273972602739725, 10.849056603773585, 12.560386473429952, 14.007782101167315, 13.488372093023257, 9.749303621169917, 16.74641148325359, 5.147058823529411, 6.643356643356643, 8.808290155440414, 13.725490196078432, 6.511627906976744, 8.635097493036211, 13.011152416356877, 12.918660287081341, 14.883720930232558, 19.913419913419915, 17.77777777777778, 9.484193011647255], 'StrongSubContext': [29.38775510204082, 29.329608938547487, 39.23705722070845, 32.66998341625207, 28.705882352941174, 29.22201138519924, 20.17738359201774, 34.15841584158416, 48.756218905472636, 53.47826086956522, 51.04602510460251, 25.83479789103691, 18.626155878467635, 48.854961832061065, 26.569037656903767, 42.3828125, 34.89583333333333, 33.06122448979592, 17.441860465116278, 19.08713692946058, 56.48535564853556, 52.30125523012552, 42.1455938697318, 38.970588235294116, 51.09170305676856, 31.80873180873181, 41.46825396825397, 45.87155963302752, 40.298507462686565, 25.793650793650798, 23.451327433628318, 23.166023166023166, 53.2258064516129, 43.7037037037037, 24.58471760797342, 27.61394101876676, 43.34975369458128, 35.294117647058826, 43.333333333333336, 23.054755043227665, 44.800000000000004, 40.35087719298245, 39.2, 39.71631205673759, 44.13145539906103, 46.46017699115044, 40.73426573426573, 39.38730853391685, 19.70954356846473, 34.8122866894198, 34.0, 37.01799485861182, 51.25858123569794, 36.12167300380228, 29.565217391304348, 36.16298811544992, 33.57664233576642, 37.22943722943723, 36.4, 31.963470319634702, 36.32075471698113, 38.164251207729464, 42.80155642023346, 46.04651162790698, 33.70473537604457, 55.980861244019145, 18.38235294117647, 22.377622377622377, 30.569948186528496, 46.40522875816993, 24.651162790697676, 30.64066852367688, 40.520446096654275, 43.54066985645933, 45.11627906976744, 58.87445887445888, 55.00000000000001, 32.11314475873544], 'WeakSub': [12.857142857142856, 20.11173184357542, 21.525885558583106, 17.24709784411277, 14.823529411764705, 13.472485768500949, 12.638580931263856, 21.782178217821784, 23.88059701492537, 16.08695652173913, 17.154811715481173, 15.465729349736378, 11.624834874504623, 12.213740458015266, 9.832635983263598, 16.2109375, 16.145833333333336, 18.775510204081634, 15.891472868217054, 20.33195020746888, 12.97071129707113, 15.481171548117153, 17.24137931034483, 17.401960784313726, 16.157205240174672, 17.463617463617464, 16.865079365079367, 9.174311926605505, 21.890547263681594, 18.650793650793652, 19.02654867256637, 12.741312741312742, 22.177419354838708, 20.74074074074074, 14.285714285714285, 15.81769436997319, 22.660098522167488, 19.607843137254903, 12.222222222222221, 15.273775216138327, 15.6, 16.95906432748538, 16.8, 19.148936170212767, 12.676056338028168, 14.601769911504425, 14.160839160839162, 17.505470459518598, 13.692946058091287, 14.675767918088736, 13.777777777777779, 13.367609254498714, 15.560640732265446, 23.193916349809886, 18.26086956521739, 16.97792869269949, 18.97810218978102, 21.645021645021643, 20.8, 22.146118721461185, 19.339622641509436, 21.73913043478261, 23.73540856031128, 22.790697674418606, 17.548746518105848, 14.832535885167463, 12.867647058823529, 17.307692307692307, 18.65284974093264, 17.647058823529413, 14.883720930232558, 11.977715877437326, 16.356877323420075, 13.397129186602871, 13.023255813953488, 12.987012987012985, 17.22222222222222, 9.983361064891847], 'WeakSubContext': [40.816326530612244, 60.05586592178771, 61.58038147138964, 51.741293532338304, 46.588235294117645, 41.36622390891841, 41.68514412416852, 58.91089108910891, 67.66169154228857, 47.82608695652174, 53.55648535564853, 46.04569420035149, 37.912813738441216, 41.603053435114504, 33.054393305439326, 50.1953125, 46.875, 56.326530612244895, 48.44961240310077, 56.43153526970954, 40.1673640167364, 46.02510460251046, 54.02298850574713, 48.77450980392157, 51.7467248908297, 51.35135135135135, 50.79365079365079, 32.11009174311927, 66.66666666666666, 52.77777777777778, 57.9646017699115, 40.15444015444015, 58.46774193548387, 60.0, 45.182724252491695, 51.206434316353885, 67.98029556650246, 55.294117647058826, 40.55555555555556, 45.53314121037464, 46.400000000000006, 48.53801169590643, 48.199999999999996, 55.319148936170215, 40.845070422535215, 43.80530973451327, 44.58041958041958, 52.516411378555794, 42.32365145228216, 44.7098976109215, 45.111111111111114, 41.38817480719794, 46.681922196796336, 63.11787072243346, 57.391304347826086, 50.93378607809848, 56.64233576642336, 57.57575757575758, 59.199999999999996, 65.29680365296804, 58.9622641509434, 62.31884057971014, 61.478599221789885, 61.86046511627907, 52.92479108635098, 50.717703349282296, 43.01470588235294, 53.84615384615385, 58.549222797927456, 52.28758169934641, 45.11627906976744, 40.947075208913645, 50.92936802973978, 44.01913875598086, 42.7906976744186, 38.52813852813853, 50.0, 34.109816971713805], 'PositiveWord': [4.489795918367347, 5.027932960893855, 4.632152588555858, 4.1459369817578775, 4.235294117647059, 4.743833017077799, 5.764966740576496, 7.425742574257425, 7.462686567164178, 4.782608695652174, 10.460251046025103, 7.205623901581721, 3.1704095112285335, 4.198473282442748, 3.765690376569038, 4.8828125, 8.854166666666668, 8.16326530612245, 4.651162790697675, 2.904564315352697, 7.531380753138076, 6.2761506276150625, 8.812260536398467, 3.6764705882352944, 6.11353711790393, 4.158004158004158, 4.166666666666666, 0.9174311926605505, 4.477611940298507, 5.555555555555555, 6.637168141592921, 5.7915057915057915, 7.258064516129033, 4.444444444444445, 3.6544850498338874, 3.2171581769436997, 3.4482758620689653, 2.7450980392156863, 4.444444444444445, 0.8645533141210375, 5.6000000000000005, 6.140350877192982, 3.4000000000000004, 4.25531914893617, 2.8169014084507045, 6.1946902654867255, 4.195804195804196, 6.564551422319474, 6.431535269709543, 6.825938566552901, 6.0, 4.884318766066838, 5.263157894736842, 6.083650190114068, 3.4782608695652173, 6.960950764006792, 5.401459854014599, 6.0606060606060606, 4.3999999999999995, 5.47945205479452, 4.245283018867925, 3.864734299516908, 8.560311284046692, 8.837209302325581, 4.178272980501393, 4.784688995215311, 1.8382352941176472, 4.545454545454546, 2.849740932642487, 7.18954248366013, 6.046511627906977, 3.064066852367688, 2.973977695167286, 7.655502392344498, 3.255813953488372, 8.225108225108226, 7.777777777777778, 3.1613976705490847], 'PositiveWordContext': [16.53061224489796, 17.039106145251395, 15.531335149863759, 15.091210613598674, 15.764705882352942, 18.40607210626186, 20.620842572062084, 25.742574257425744, 24.875621890547265, 16.52173913043478, 36.40167364016737, 24.78031634446397, 11.756935270805812, 16.793893129770993, 14.644351464435147, 18.1640625, 30.729166666666668, 28.97959183673469, 17.441860465116278, 11.618257261410788, 27.615062761506277, 24.267782426778243, 31.800766283524908, 13.23529411764706, 22.270742358078603, 15.592515592515593, 15.079365079365079, 3.669724770642202, 16.417910447761194, 19.444444444444446, 23.893805309734514, 19.69111969111969, 24.596774193548388, 14.814814814814813, 13.621262458471762, 12.600536193029491, 13.30049261083744, 10.588235294117647, 15.0, 3.45821325648415, 20.8, 18.71345029239766, 11.4, 14.184397163120568, 11.267605633802818, 21.238938053097346, 15.909090909090908, 23.194748358862142, 23.029045643153527, 24.2320819112628, 22.22222222222222, 17.480719794344473, 19.221967963386728, 22.433460076045627, 11.304347826086957, 23.599320882852293, 19.562043795620436, 22.943722943722943, 14.000000000000002, 20.091324200913242, 16.037735849056602, 15.458937198067632, 30.739299610894943, 28.837209302325583, 14.763231197771587, 17.22488038277512, 7.352941176470589, 17.307692307692307, 10.880829015544041, 26.143790849673206, 20.0, 10.86350974930362, 11.152416356877323, 24.880382775119617, 12.093023255813954, 26.83982683982684, 25.555555555555554, 12.312811980033278], 'NegativeWord': [4.285714285714286, 2.5139664804469275, 10.08174386920981, 5.638474295190713, 4.705882352941177, 5.1233396584440225, 3.5476718403547673, 3.9603960396039604, 10.945273631840797, 9.130434782608695, 9.205020920502092, 5.975395430579964, 5.019815059445178, 8.778625954198473, 5.439330543933055, 7.8125, 8.333333333333332, 7.755102040816326, 0.3875968992248062, 2.0746887966804977, 12.552301255230125, 11.297071129707113, 9.195402298850574, 10.784313725490197, 13.973799126637553, 4.5738045738045745, 8.531746031746032, 10.091743119266056, 8.955223880597014, 6.746031746031746, 5.752212389380531, 4.633204633204633, 9.67741935483871, 4.444444444444445, 5.3156146179401995, 3.485254691689008, 15.763546798029557, 14.50980392156863, 12.777777777777777, 6.340057636887608, 11.200000000000001, 11.695906432748536, 13.600000000000001, 14.184397163120568, 16.431924882629108, 7.079646017699115, 7.167832167832168, 10.065645514223196, 2.0746887966804977, 4.436860068259386, 5.333333333333334, 6.169665809768637, 11.899313501144166, 12.927756653992395, 13.043478260869565, 9.337860780984721, 9.05109489051095, 11.688311688311687, 14.399999999999999, 10.50228310502283, 11.79245283018868, 14.009661835748794, 11.284046692607005, 9.767441860465116, 8.913649025069638, 13.875598086124402, 6.61764705882353, 8.216783216783217, 12.694300518134716, 4.57516339869281, 5.5813953488372094, 6.685236768802229, 12.639405204460965, 6.220095693779904, 11.627906976744185, 11.688311688311687, 11.11111111111111, 5.657237936772046], 'NegativeWordContext': [15.918367346938775, 9.217877094972067, 32.42506811989101, 20.8955223880597, 17.88235294117647, 18.785578747628083, 13.082039911308204, 13.861386138613863, 37.3134328358209, 31.73913043478261, 30.543933054393307, 21.26537785588752, 17.701453104359313, 27.099236641221374, 19.874476987447697, 26.953125, 22.916666666666664, 28.163265306122447, 1.550387596899225, 8.29875518672199, 43.93305439330544, 37.65690376569037, 32.56704980842912, 36.27450980392157, 45.633187772925766, 16.216216216216218, 29.365079365079367, 32.11009174311927, 33.33333333333333, 25.396825396825395, 21.238938053097346, 17.374517374517374, 31.85483870967742, 17.037037037037038, 18.272425249169437, 12.600536193029491, 53.20197044334976, 49.01960784313725, 41.66666666666667, 21.902017291066283, 36.4, 38.59649122807017, 41.8, 36.87943262411347, 50.70422535211267, 26.991150442477874, 25.524475524475527, 35.44857768052516, 7.053941908713693, 16.38225255972696, 19.333333333333332, 23.39331619537275, 39.130434782608695, 42.585551330798474, 44.34782608695652, 33.106960950764005, 30.802919708029197, 39.39393939393939, 44.4, 36.757990867579906, 40.56603773584906, 40.57971014492754, 35.019455252918284, 30.23255813953488, 30.362116991643457, 46.411483253588514, 23.897058823529413, 29.895104895104897, 43.78238341968912, 16.99346405228758, 20.930232558139537, 24.512534818941504, 41.63568773234201, 21.052631578947366, 38.139534883720934, 38.961038961038966, 39.44444444444444, 19.467554076539102], 'BiasLexicon': [24.897959183673468, 24.022346368715084, 25.340599455040874, 29.684908789386398, 26.35294117647059, 24.09867172675522, 26.82926829268293, 29.207920792079207, 28.35820895522388, 31.30434782608696, 26.359832635983267, 20.913884007029875, 21.136063408190225, 25.954198473282442, 22.594142259414227, 23.046875, 30.729166666666668, 27.346938775510203, 25.968992248062015, 29.045643153526974, 25.10460251046025, 21.75732217573222, 26.436781609195403, 28.186274509803923, 25.54585152838428, 27.650727650727653, 26.58730158730159, 22.93577981651376, 30.34825870646766, 28.57142857142857, 30.08849557522124, 26.254826254826252, 30.241935483870968, 31.11111111111111, 22.92358803986711, 30.831099195710454, 26.108374384236456, 35.294117647058826, 20.0, 28.818443804034583, 25.6, 28.07017543859649, 23.400000000000002, 29.78723404255319, 30.51643192488263, 26.991150442477874, 31.993006993006993, 25.601750547045953, 15.975103734439832, 26.621160409556317, 22.666666666666664, 26.221079691516707, 18.53546910755149, 29.277566539923956, 19.130434782608695, 23.2597623089983, 24.379562043795623, 30.303030303030305, 24.0, 26.48401826484018, 28.77358490566038, 31.40096618357488, 26.848249027237355, 30.23255813953488, 29.805013927576603, 31.100478468899524, 14.705882352941178, 20.454545454545457, 23.05699481865285, 18.30065359477124, 29.30232558139535, 25.62674094707521, 28.624535315985128, 20.095693779904305, 23.72093023255814, 28.13852813852814, 31.11111111111111, 23.128119800332776]}\n"
     ]
    }
   ],
   "source": [
    "'''for i in range(data.read_data().shape[0]):\n",
    "    article=data.clean_dataset()['Article'][i]'''\n",
    "result={\n",
    "        'Hedge':[],\n",
    "        'HedgeContext':[],\n",
    "        'FativeVerb':[],\n",
    "        'FactiveVerbContext':[],\n",
    "        'AssertiveVerb':[],\n",
    "        'AssertiveVerbContext':[],\n",
    "        'ImplicativeVerb':[],\n",
    "        'ImplicativeVerbContext':[],\n",
    "        'ReportVerb':[],\n",
    "        'ReportVerbContext':[],\n",
    "        'StrongSub':[],\n",
    "        'StrongSubContext':[],\n",
    "        'WeakSub':[],\n",
    "        'WeakSubContext':[],\n",
    "        'PositiveWord':[],\n",
    "        'PositiveWordContext':[],\n",
    "        'NegativeWord':[],\n",
    "        'NegativeWordContext':[],\n",
    "        'BiasLexicon':[]\n",
    "}\n",
    "size=data.read_data().shape[0]\n",
    "for j in range(size):\n",
    "    #data.clean_dataset()['Article'][i]\n",
    "    article=data.clean_dataset()['Article'][j]\n",
    "    tokens=word_tokenize(article)\n",
    "    stopwordsInArticle,stopwordsNotInArticle=remove_stop_words(tokens)\n",
    "    stopwordsNotInArticle=lemmatise(stopwordsNotInArticle)\n",
    "    features = {\n",
    "            'word':[],\n",
    "            'POS':[],\n",
    "            'POSNeg1':[],\n",
    "            'POSNeg2':[],\n",
    "            'POS1':[],\n",
    "            'POS2':[],\n",
    "            'Hedge':[],\n",
    "            'HedgeContext':[],\n",
    "            'FativeVerb':[],\n",
    "            'FactiveVerbContext':[],\n",
    "            'AssertiveVerb':[],\n",
    "            'AssertiveVerbContext':[],\n",
    "            'ImplicativeVerb':[],\n",
    "            'ImplicativeVerbContext':[],\n",
    "            'ReportVerb':[],\n",
    "            'ReportVerbContext':[],\n",
    "            'StrongSub':[],\n",
    "            'StrongSubContext':[],\n",
    "            'WeakSub':[],\n",
    "            'WeakSubContext':[],\n",
    "            'PositiveWord':[],\n",
    "            'PositiveWordContext':[],\n",
    "            'NegativeWord':[],\n",
    "            'NegativeWordContext':[],\n",
    "            'BiasLexicon':[]\n",
    "    }\n",
    "\n",
    "    for i in range(len(stopwordsNotInArticle)):\n",
    "        features['word'].append(stopwordsNotInArticle[i])\n",
    "        features['POS'].append(pos(stopwordsNotInArticle[i]))\n",
    "        features['POSNeg1'].append(posNeg1(stopwordsNotInArticle,i))\n",
    "        features['POSNeg2'].append(posNeg2(stopwordsNotInArticle,i))\n",
    "        features['POS1'].append(pos1(stopwordsNotInArticle,i))\n",
    "        features['POS2'].append(pos2(stopwordsNotInArticle,i))\n",
    "        features['Hedge'].append(hedges(stopwordsNotInArticle[i]))\n",
    "        features['HedgeContext'].append(hedges_context(stopwordsNotInArticle,stopwordsNotInArticle[i],i))\n",
    "        features['FativeVerb'].append(factives_hooper(stopwordsNotInArticle[i]))\n",
    "        features['FactiveVerbContext'].append(factives_hooper_context(stopwordsNotInArticle,stopwordsNotInArticle[i],i))\n",
    "        features['AssertiveVerb'].append(assertive_hooper(stopwordsNotInArticle[i]))\n",
    "        features['AssertiveVerbContext'].append(assertive_context(stopwordsNotInArticle,stopwordsNotInArticle[i],i))\n",
    "        features['ImplicativeVerb'].append(implicative_verb(stopwordsNotInArticle[i]))\n",
    "        features['ImplicativeVerbContext'].append(implicative_verb_context(stopwordsNotInArticle,stopwordsNotInArticle[i],i))\n",
    "        features['ReportVerb'].append(report_verb(stopwordsNotInArticle[i]))\n",
    "        features['ReportVerbContext'].append(report_verb_context(stopwordsNotInArticle,stopwordsNotInArticle[i],i))\n",
    "        features['StrongSub'].append(strongSubjectivtiy(stopwordsNotInArticle[i]))\n",
    "        features['StrongSubContext'].append(strongSubjectivityContext(stopwordsNotInArticle,stopwordsNotInArticle[i],i))\n",
    "        features['WeakSub'].append(weakSubjectivtiy(stopwordsNotInArticle[i]))\n",
    "        features['WeakSubContext'].append(weakSubjectivityContext(stopwordsNotInArticle,stopwordsNotInArticle[i],i))\n",
    "        features['PositiveWord'].append(positivewords(stopwordsNotInArticle[i]))\n",
    "        features['PositiveWordContext'].append(positiveWordContext(stopwordsNotInArticle,stopwordsNotInArticle[i],i))\n",
    "        features['NegativeWord'].append(negativewords(stopwordsNotInArticle[i]))\n",
    "        features['NegativeWordContext'].append(negativeWordContext(stopwordsNotInArticle,stopwordsNotInArticle[i],i))\n",
    "        features['BiasLexicon'].append(biasLexion(stopwordsNotInArticle[i]))\n",
    "\n",
    "    #features\n",
    "    df = pd.DataFrame(features, columns = [\n",
    "            'word',\n",
    "            'POS',\n",
    "            'POSNeg1',\n",
    "            'POSNeg2',\n",
    "            'POS1',\n",
    "            'POS2',\n",
    "            'Hedge',\n",
    "            'HedgeContext',\n",
    "            'FativeVerb',\n",
    "            'FactiveVerbContext',\n",
    "            'AssertiveVerb',\n",
    "            'AssertiveVerbContext',\n",
    "            'ImplicativeVerb',\n",
    "            'ImplicativeVerbContext',\n",
    "            'ReportVerb',\n",
    "            'ReportVerbContext',\n",
    "            'StrongSub',\n",
    "            'StrongSubContext',\n",
    "            'WeakSub',\n",
    "            'WeakSubContext',\n",
    "            'PositiveWord',\n",
    "            'PositiveWordContext',\n",
    "            'NegativeWord',\n",
    "            'NegativeWordContext',\n",
    "            'BiasLexicon']\n",
    "            )\n",
    "\n",
    "    for col in df.columns:\n",
    "        if col!='word' and col!='POS' and col!='POSNeg1' and col!='POSNeg2' and col!='POS1' and col!='POS2':\n",
    "            df[col]=df[col]*1\n",
    "    r,c = df.shape\n",
    "\n",
    "    for col in df.columns:\n",
    "        if col!='word' and col!='POS' and col!='POSNeg1' and col!='POSNeg2' and col!='POS1' and col!='POS2':\n",
    "            result[col].append((df[col].sum()/r)*100)\n",
    "            \n",
    "            \n",
    "print(result)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 307,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Hedge</th>\n",
       "      <th>HedgeContext</th>\n",
       "      <th>FativeVerb</th>\n",
       "      <th>FactiveVerbContext</th>\n",
       "      <th>AssertiveVerb</th>\n",
       "      <th>AssertiveVerbContext</th>\n",
       "      <th>ImplicativeVerb</th>\n",
       "      <th>ImplicativeVerbContext</th>\n",
       "      <th>ReportVerb</th>\n",
       "      <th>ReportVerbContext</th>\n",
       "      <th>StrongSub</th>\n",
       "      <th>StrongSubContext</th>\n",
       "      <th>WeakSub</th>\n",
       "      <th>WeakSubContext</th>\n",
       "      <th>PositiveWord</th>\n",
       "      <th>PositiveWordContext</th>\n",
       "      <th>NegativeWord</th>\n",
       "      <th>NegativeWordContext</th>\n",
       "      <th>BiasLexicon</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.836735</td>\n",
       "      <td>6.938776</td>\n",
       "      <td>0.612245</td>\n",
       "      <td>2.448980</td>\n",
       "      <td>4.489796</td>\n",
       "      <td>17.346939</td>\n",
       "      <td>0.612245</td>\n",
       "      <td>2.448980</td>\n",
       "      <td>7.346939</td>\n",
       "      <td>27.551020</td>\n",
       "      <td>8.979592</td>\n",
       "      <td>29.387755</td>\n",
       "      <td>12.857143</td>\n",
       "      <td>40.816327</td>\n",
       "      <td>4.489796</td>\n",
       "      <td>16.530612</td>\n",
       "      <td>4.285714</td>\n",
       "      <td>15.918367</td>\n",
       "      <td>24.897959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.910615</td>\n",
       "      <td>14.245810</td>\n",
       "      <td>0.279330</td>\n",
       "      <td>1.117318</td>\n",
       "      <td>3.072626</td>\n",
       "      <td>12.290503</td>\n",
       "      <td>1.117318</td>\n",
       "      <td>4.469274</td>\n",
       "      <td>6.145251</td>\n",
       "      <td>22.625698</td>\n",
       "      <td>8.659218</td>\n",
       "      <td>29.329609</td>\n",
       "      <td>20.111732</td>\n",
       "      <td>60.055866</td>\n",
       "      <td>5.027933</td>\n",
       "      <td>17.039106</td>\n",
       "      <td>2.513966</td>\n",
       "      <td>9.217877</td>\n",
       "      <td>24.022346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2.452316</td>\n",
       "      <td>9.809264</td>\n",
       "      <td>0.544959</td>\n",
       "      <td>1.907357</td>\n",
       "      <td>4.359673</td>\n",
       "      <td>17.166213</td>\n",
       "      <td>0.817439</td>\n",
       "      <td>3.269755</td>\n",
       "      <td>4.632153</td>\n",
       "      <td>18.256131</td>\n",
       "      <td>13.079019</td>\n",
       "      <td>39.237057</td>\n",
       "      <td>21.525886</td>\n",
       "      <td>61.580381</td>\n",
       "      <td>4.632153</td>\n",
       "      <td>15.531335</td>\n",
       "      <td>10.081744</td>\n",
       "      <td>32.425068</td>\n",
       "      <td>25.340599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4.643449</td>\n",
       "      <td>17.744610</td>\n",
       "      <td>1.824212</td>\n",
       "      <td>7.131012</td>\n",
       "      <td>5.970149</td>\n",
       "      <td>22.553897</td>\n",
       "      <td>0.497512</td>\n",
       "      <td>1.824212</td>\n",
       "      <td>7.960199</td>\n",
       "      <td>28.192371</td>\n",
       "      <td>10.281924</td>\n",
       "      <td>32.669983</td>\n",
       "      <td>17.247098</td>\n",
       "      <td>51.741294</td>\n",
       "      <td>4.145937</td>\n",
       "      <td>15.091211</td>\n",
       "      <td>5.638474</td>\n",
       "      <td>20.895522</td>\n",
       "      <td>29.684909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.411765</td>\n",
       "      <td>4.705882</td>\n",
       "      <td>0.235294</td>\n",
       "      <td>0.941176</td>\n",
       "      <td>3.294118</td>\n",
       "      <td>11.764706</td>\n",
       "      <td>0.941176</td>\n",
       "      <td>3.764706</td>\n",
       "      <td>6.823529</td>\n",
       "      <td>24.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>28.705882</td>\n",
       "      <td>14.823529</td>\n",
       "      <td>46.588235</td>\n",
       "      <td>4.235294</td>\n",
       "      <td>15.764706</td>\n",
       "      <td>4.705882</td>\n",
       "      <td>17.882353</td>\n",
       "      <td>26.352941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>73</td>\n",
       "      <td>2.392344</td>\n",
       "      <td>9.569378</td>\n",
       "      <td>0.956938</td>\n",
       "      <td>3.827751</td>\n",
       "      <td>2.392344</td>\n",
       "      <td>9.569378</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.263158</td>\n",
       "      <td>20.574163</td>\n",
       "      <td>12.918660</td>\n",
       "      <td>43.540670</td>\n",
       "      <td>13.397129</td>\n",
       "      <td>44.019139</td>\n",
       "      <td>7.655502</td>\n",
       "      <td>24.880383</td>\n",
       "      <td>6.220096</td>\n",
       "      <td>21.052632</td>\n",
       "      <td>20.095694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>74</td>\n",
       "      <td>0.465116</td>\n",
       "      <td>1.860465</td>\n",
       "      <td>0.930233</td>\n",
       "      <td>3.720930</td>\n",
       "      <td>2.325581</td>\n",
       "      <td>9.302326</td>\n",
       "      <td>0.930233</td>\n",
       "      <td>3.720930</td>\n",
       "      <td>6.511628</td>\n",
       "      <td>23.720930</td>\n",
       "      <td>14.883721</td>\n",
       "      <td>45.116279</td>\n",
       "      <td>13.023256</td>\n",
       "      <td>42.790698</td>\n",
       "      <td>3.255814</td>\n",
       "      <td>12.093023</td>\n",
       "      <td>11.627907</td>\n",
       "      <td>38.139535</td>\n",
       "      <td>23.720930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>75</td>\n",
       "      <td>2.164502</td>\n",
       "      <td>8.225108</td>\n",
       "      <td>0.865801</td>\n",
       "      <td>3.463203</td>\n",
       "      <td>3.896104</td>\n",
       "      <td>15.151515</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>9.090909</td>\n",
       "      <td>34.632035</td>\n",
       "      <td>19.913420</td>\n",
       "      <td>58.874459</td>\n",
       "      <td>12.987013</td>\n",
       "      <td>38.528139</td>\n",
       "      <td>8.225108</td>\n",
       "      <td>26.839827</td>\n",
       "      <td>11.688312</td>\n",
       "      <td>38.961039</td>\n",
       "      <td>28.138528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>76</td>\n",
       "      <td>1.666667</td>\n",
       "      <td>6.666667</td>\n",
       "      <td>2.777778</td>\n",
       "      <td>11.111111</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>18.333333</td>\n",
       "      <td>2.222222</td>\n",
       "      <td>8.888889</td>\n",
       "      <td>11.111111</td>\n",
       "      <td>37.777778</td>\n",
       "      <td>17.777778</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>17.222222</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>7.777778</td>\n",
       "      <td>25.555556</td>\n",
       "      <td>11.111111</td>\n",
       "      <td>39.444444</td>\n",
       "      <td>31.111111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>77</td>\n",
       "      <td>1.331115</td>\n",
       "      <td>4.991681</td>\n",
       "      <td>1.996672</td>\n",
       "      <td>7.653910</td>\n",
       "      <td>5.158070</td>\n",
       "      <td>19.966722</td>\n",
       "      <td>0.665557</td>\n",
       "      <td>2.662230</td>\n",
       "      <td>10.981697</td>\n",
       "      <td>37.770383</td>\n",
       "      <td>9.484193</td>\n",
       "      <td>32.113145</td>\n",
       "      <td>9.983361</td>\n",
       "      <td>34.109817</td>\n",
       "      <td>3.161398</td>\n",
       "      <td>12.312812</td>\n",
       "      <td>5.657238</td>\n",
       "      <td>19.467554</td>\n",
       "      <td>23.128120</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>78 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Hedge  HedgeContext  FativeVerb  FactiveVerbContext  AssertiveVerb  \\\n",
       "0   1.836735      6.938776    0.612245            2.448980       4.489796   \n",
       "1   3.910615     14.245810    0.279330            1.117318       3.072626   \n",
       "2   2.452316      9.809264    0.544959            1.907357       4.359673   \n",
       "3   4.643449     17.744610    1.824212            7.131012       5.970149   \n",
       "4   1.411765      4.705882    0.235294            0.941176       3.294118   \n",
       "..       ...           ...         ...                 ...            ...   \n",
       "73  2.392344      9.569378    0.956938            3.827751       2.392344   \n",
       "74  0.465116      1.860465    0.930233            3.720930       2.325581   \n",
       "75  2.164502      8.225108    0.865801            3.463203       3.896104   \n",
       "76  1.666667      6.666667    2.777778           11.111111       5.000000   \n",
       "77  1.331115      4.991681    1.996672            7.653910       5.158070   \n",
       "\n",
       "    AssertiveVerbContext  ImplicativeVerb  ImplicativeVerbContext  ReportVerb  \\\n",
       "0              17.346939         0.612245                2.448980    7.346939   \n",
       "1              12.290503         1.117318                4.469274    6.145251   \n",
       "2              17.166213         0.817439                3.269755    4.632153   \n",
       "3              22.553897         0.497512                1.824212    7.960199   \n",
       "4              11.764706         0.941176                3.764706    6.823529   \n",
       "..                   ...              ...                     ...         ...   \n",
       "73              9.569378         0.000000                0.000000    5.263158   \n",
       "74              9.302326         0.930233                3.720930    6.511628   \n",
       "75             15.151515         0.000000                0.000000    9.090909   \n",
       "76             18.333333         2.222222                8.888889   11.111111   \n",
       "77             19.966722         0.665557                2.662230   10.981697   \n",
       "\n",
       "    ReportVerbContext  StrongSub  StrongSubContext    WeakSub  WeakSubContext  \\\n",
       "0           27.551020   8.979592         29.387755  12.857143       40.816327   \n",
       "1           22.625698   8.659218         29.329609  20.111732       60.055866   \n",
       "2           18.256131  13.079019         39.237057  21.525886       61.580381   \n",
       "3           28.192371  10.281924         32.669983  17.247098       51.741294   \n",
       "4           24.000000   8.000000         28.705882  14.823529       46.588235   \n",
       "..                ...        ...               ...        ...             ...   \n",
       "73          20.574163  12.918660         43.540670  13.397129       44.019139   \n",
       "74          23.720930  14.883721         45.116279  13.023256       42.790698   \n",
       "75          34.632035  19.913420         58.874459  12.987013       38.528139   \n",
       "76          37.777778  17.777778         55.000000  17.222222       50.000000   \n",
       "77          37.770383   9.484193         32.113145   9.983361       34.109817   \n",
       "\n",
       "    PositiveWord  PositiveWordContext  NegativeWord  NegativeWordContext  \\\n",
       "0       4.489796            16.530612      4.285714            15.918367   \n",
       "1       5.027933            17.039106      2.513966             9.217877   \n",
       "2       4.632153            15.531335     10.081744            32.425068   \n",
       "3       4.145937            15.091211      5.638474            20.895522   \n",
       "4       4.235294            15.764706      4.705882            17.882353   \n",
       "..           ...                  ...           ...                  ...   \n",
       "73      7.655502            24.880383      6.220096            21.052632   \n",
       "74      3.255814            12.093023     11.627907            38.139535   \n",
       "75      8.225108            26.839827     11.688312            38.961039   \n",
       "76      7.777778            25.555556     11.111111            39.444444   \n",
       "77      3.161398            12.312812      5.657238            19.467554   \n",
       "\n",
       "    BiasLexicon  \n",
       "0     24.897959  \n",
       "1     24.022346  \n",
       "2     25.340599  \n",
       "3     29.684909  \n",
       "4     26.352941  \n",
       "..          ...  \n",
       "73    20.095694  \n",
       "74    23.720930  \n",
       "75    28.138528  \n",
       "76    31.111111  \n",
       "77    23.128120  \n",
       "\n",
       "[78 rows x 19 columns]"
      ]
     },
     "execution_count": 307,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_result = pd.DataFrame(result, columns = [\n",
    "        'Hedge',\n",
    "        'HedgeContext',\n",
    "        'FativeVerb',\n",
    "        'FactiveVerbContext',\n",
    "        'AssertiveVerb',\n",
    "        'AssertiveVerbContext',\n",
    "        'ImplicativeVerb',\n",
    "        'ImplicativeVerbContext',\n",
    "        'ReportVerb',\n",
    "        'ReportVerbContext',\n",
    "        'StrongSub',\n",
    "        'StrongSubContext',\n",
    "        'WeakSub',\n",
    "        'WeakSubContext',\n",
    "        'PositiveWord',\n",
    "        'PositiveWordContext',\n",
    "        'NegativeWord',\n",
    "        'NegativeWordContext',\n",
    "        'BiasLexicon']\n",
    "        )\n",
    "df_result.to_csv('result.csv')\n",
    "df_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>POS</th>\n",
       "      <th>POSNeg1</th>\n",
       "      <th>POSNeg2</th>\n",
       "      <th>POS1</th>\n",
       "      <th>POS2</th>\n",
       "      <th>Hedge</th>\n",
       "      <th>HedgeContext</th>\n",
       "      <th>FativeVerb</th>\n",
       "      <th>FactiveVerbContext</th>\n",
       "      <th>...</th>\n",
       "      <th>ReportVerbContext</th>\n",
       "      <th>StrongSub</th>\n",
       "      <th>StrongSubContext</th>\n",
       "      <th>WeakSub</th>\n",
       "      <th>WeakSubContext</th>\n",
       "      <th>PositiveWord</th>\n",
       "      <th>PositiveWordContext</th>\n",
       "      <th>NegativeWord</th>\n",
       "      <th>NegativeWordContext</th>\n",
       "      <th>BiasLexicon</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>preparations</td>\n",
       "      <td>NNS</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>ram</td>\n",
       "      <td>NN</td>\n",
       "      <td>NNS</td>\n",
       "      <td>none</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>mandir</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NNS</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>bhoomi</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>poojan</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>289</td>\n",
       "      <td>construction</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>VB</td>\n",
       "      <td>VB</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>290</td>\n",
       "      <td>take</td>\n",
       "      <td>VB</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>CD</td>\n",
       "      <td>CD</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>291</td>\n",
       "      <td>six</td>\n",
       "      <td>CD</td>\n",
       "      <td>VB</td>\n",
       "      <td>NN</td>\n",
       "      <td>NNS</td>\n",
       "      <td>NNS</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>292</td>\n",
       "      <td>months</td>\n",
       "      <td>NNS</td>\n",
       "      <td>CD</td>\n",
       "      <td>VB</td>\n",
       "      <td>NN</td>\n",
       "      <td>none</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>293</td>\n",
       "      <td>year</td>\n",
       "      <td>NN</td>\n",
       "      <td>NNS</td>\n",
       "      <td>CD</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>294 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             word  POS POSNeg1 POSNeg2  POS1  POS2  Hedge  HedgeContext  \\\n",
       "0    preparations  NNS    none    none    NN    NN  False         False   \n",
       "1             ram   NN     NNS    none    NN    NN  False         False   \n",
       "2          mandir   NN      NN     NNS    NN    NN  False         False   \n",
       "3          bhoomi   NN      NN      NN    NN    NN  False         False   \n",
       "4          poojan   NN      NN      NN    NN    NN  False         False   \n",
       "..            ...  ...     ...     ...   ...   ...    ...           ...   \n",
       "289  construction   NN      NN      NN    VB    VB  False          True   \n",
       "290          take   VB      NN      NN    CD    CD  False          True   \n",
       "291           six   CD      VB      NN   NNS   NNS  False         False   \n",
       "292        months  NNS      CD      VB    NN  none  False         False   \n",
       "293          year   NN     NNS      CD  none  none  False         False   \n",
       "\n",
       "     FativeVerb  FactiveVerbContext  ...  ReportVerbContext  StrongSub  \\\n",
       "0         False               False  ...              False      False   \n",
       "1         False               False  ...              False      False   \n",
       "2         False               False  ...              False      False   \n",
       "3         False               False  ...              False      False   \n",
       "4         False               False  ...              False      False   \n",
       "..          ...                 ...  ...                ...        ...   \n",
       "289       False               False  ...               True      False   \n",
       "290       False               False  ...               True      False   \n",
       "291       False               False  ...              False      False   \n",
       "292       False               False  ...              False      False   \n",
       "293       False               False  ...              False      False   \n",
       "\n",
       "     StrongSubContext  WeakSub  WeakSubContext  PositiveWord  \\\n",
       "0               False    False           False         False   \n",
       "1               False    False           False         False   \n",
       "2               False    False           False         False   \n",
       "3               False    False           False         False   \n",
       "4               False    False           False         False   \n",
       "..                ...      ...             ...           ...   \n",
       "289             False    False            True         False   \n",
       "290             False    False            True         False   \n",
       "291             False    False           False         False   \n",
       "292             False    False           False         False   \n",
       "293             False    False           False         False   \n",
       "\n",
       "     PositiveWordContext  NegativeWord  NegativeWordContext  BiasLexicon  \n",
       "0                  False         False                False        False  \n",
       "1                  False         False                False        False  \n",
       "2                  False         False                False        False  \n",
       "3                  False         False                False        False  \n",
       "4                  False         False                False        False  \n",
       "..                   ...           ...                  ...          ...  \n",
       "289                 True         False                False        False  \n",
       "290                False         False                False         True  \n",
       "291                False         False                False        False  \n",
       "292                False         False                False        False  \n",
       "293                False         False                False         True  \n",
       "\n",
       "[294 rows x 25 columns]"
      ]
     },
     "execution_count": 248,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(features, columns = [\n",
    "        'word',\n",
    "        'POS',\n",
    "        'POSNeg1',\n",
    "        'POSNeg2',\n",
    "        'POS1',\n",
    "        'POS2',\n",
    "        'Hedge',\n",
    "        'HedgeContext',\n",
    "        'FativeVerb',\n",
    "        'FactiveVerbContext',\n",
    "        'AssertiveVerb',\n",
    "        'AssertiveVerbContext',\n",
    "        'ImplicativeVerb',\n",
    "        'ImplicativeVerbContext',\n",
    "        'ReportVerb',\n",
    "        'ReportVerbContext',\n",
    "        'StrongSub',\n",
    "        'StrongSubContext',\n",
    "        'WeakSub',\n",
    "        'WeakSubContext',\n",
    "        'PositiveWord',\n",
    "        'PositiveWordContext',\n",
    "        'NegativeWord',\n",
    "        'NegativeWordContext',\n",
    "        'BiasLexicon']\n",
    "        )\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "word                      object\n",
       "POS                       object\n",
       "POSNeg1                   object\n",
       "POSNeg2                   object\n",
       "POS1                      object\n",
       "POS2                      object\n",
       "Hedge                       bool\n",
       "HedgeContext                bool\n",
       "FativeVerb                  bool\n",
       "FactiveVerbContext          bool\n",
       "AssertiveVerb               bool\n",
       "AssertiveVerbContext        bool\n",
       "ImplicativeVerb             bool\n",
       "ImplicativeVerbContext      bool\n",
       "ReportVerb                  bool\n",
       "ReportVerbContext           bool\n",
       "StrongSub                   bool\n",
       "StrongSubContext            bool\n",
       "WeakSub                     bool\n",
       "WeakSubContext              bool\n",
       "PositiveWord                bool\n",
       "PositiveWordContext         bool\n",
       "NegativeWord                bool\n",
       "NegativeWordContext         bool\n",
       "BiasLexicon                 bool\n",
       "dtype: object"
      ]
     },
     "execution_count": 272,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "#converting to int values\n",
    "\n",
    "'''df['Hedge']=df['Hedge'] * 1\n",
    "df['HedgeContext']=df['HedgeContext'] * 1\n",
    "df['FativeVerb']=df['FativeVerb'] * 1\n",
    "df['FactiveVerbContext']=df['FactiveVerbContext'] * 1\n",
    "df['AssertiveVerb']=df['AssertiveVerb'] * 1\n",
    "df['AssertiveVerbContext']=df['AssertiveVerbContext'] * 1\n",
    "df['ImplicativeVerb']=df['ImplicativeVerb']*1\n",
    "df['ImplicativeVerbContext']=df['ImplicativeVerbContext'] * 1\n",
    "df['ReportVerb']=df['ReportVerb'] * 1\n",
    "df['ReportVerbContext'] = df['ReportVerbContext'] * 1\n",
    "df['StrongSub']=df['StrongSub'] * 1\n",
    "df['StrongSubContext']=df['StrongSubContext'] * 1\n",
    "df['WeakSub'] = df['WeakSub'] * 1\n",
    "df['WeakSubContext'] = df['WeakSubContext'] * 1\n",
    "df['PositiveWord'] = df['PositiveWord'] * 1\n",
    "df['PositiveWordContext'] = df['PositiveWordContext'] * 1\n",
    "df['NegativeWord']  = df['NegativeWord'] * 1\n",
    "df['NegativeWordContext'] = df['NegativeWordContext'] * 1\n",
    "df['BiasLexicon'] = df['BiasLexicon'] * 1'''\n",
    "for col in df.columns:\n",
    "    if col!='word' and col!='POS' and col!='POSNeg1' and col!='POSNeg2' and col!='POS1' and col!='POS2':\n",
    "        df[col]=df[col]*1\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "word                      object\n",
       "POS                       object\n",
       "POSNeg1                   object\n",
       "POSNeg2                   object\n",
       "POS1                      object\n",
       "POS2                      object\n",
       "Hedge                      int32\n",
       "HedgeContext               int32\n",
       "FativeVerb                 int32\n",
       "FactiveVerbContext         int32\n",
       "AssertiveVerb              int32\n",
       "AssertiveVerbContext       int32\n",
       "ImplicativeVerb            int32\n",
       "ImplicativeVerbContext     int32\n",
       "ReportVerb                 int32\n",
       "ReportVerbContext          int32\n",
       "StrongSub                  int32\n",
       "StrongSubContext           int32\n",
       "WeakSub                    int32\n",
       "WeakSubContext             int32\n",
       "PositiveWord               int32\n",
       "PositiveWordContext        int32\n",
       "NegativeWord               int32\n",
       "NegativeWordContext        int32\n",
       "BiasLexicon                int32\n",
       "dtype: object"
      ]
     },
     "execution_count": 274,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>word</th>\n",
       "      <th>POS</th>\n",
       "      <th>POSNeg1</th>\n",
       "      <th>POSNeg2</th>\n",
       "      <th>POS1</th>\n",
       "      <th>POS2</th>\n",
       "      <th>Hedge</th>\n",
       "      <th>HedgeContext</th>\n",
       "      <th>FativeVerb</th>\n",
       "      <th>FactiveVerbContext</th>\n",
       "      <th>...</th>\n",
       "      <th>ReportVerbContext</th>\n",
       "      <th>StrongSub</th>\n",
       "      <th>StrongSubContext</th>\n",
       "      <th>WeakSub</th>\n",
       "      <th>WeakSubContext</th>\n",
       "      <th>PositiveWord</th>\n",
       "      <th>PositiveWordContext</th>\n",
       "      <th>NegativeWord</th>\n",
       "      <th>NegativeWordContext</th>\n",
       "      <th>BiasLexicon</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>preparations</td>\n",
       "      <td>NNS</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>ram</td>\n",
       "      <td>NN</td>\n",
       "      <td>NNS</td>\n",
       "      <td>none</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>mandir</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NNS</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>bhoomi</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>poojan</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>583</td>\n",
       "      <td>construction</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>VB</td>\n",
       "      <td>VB</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>584</td>\n",
       "      <td>take</td>\n",
       "      <td>VB</td>\n",
       "      <td>NN</td>\n",
       "      <td>NN</td>\n",
       "      <td>CD</td>\n",
       "      <td>CD</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>585</td>\n",
       "      <td>six</td>\n",
       "      <td>CD</td>\n",
       "      <td>VB</td>\n",
       "      <td>NN</td>\n",
       "      <td>NNS</td>\n",
       "      <td>NNS</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>586</td>\n",
       "      <td>months</td>\n",
       "      <td>NNS</td>\n",
       "      <td>CD</td>\n",
       "      <td>VB</td>\n",
       "      <td>NN</td>\n",
       "      <td>none</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>587</td>\n",
       "      <td>year</td>\n",
       "      <td>NN</td>\n",
       "      <td>NNS</td>\n",
       "      <td>CD</td>\n",
       "      <td>none</td>\n",
       "      <td>none</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>588 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             word  POS POSNeg1 POSNeg2  POS1  POS2  Hedge  HedgeContext  \\\n",
       "0    preparations  NNS    none    none    NN    NN      0             0   \n",
       "1             ram   NN     NNS    none    NN    NN      0             0   \n",
       "2          mandir   NN      NN     NNS    NN    NN      0             0   \n",
       "3          bhoomi   NN      NN      NN    NN    NN      0             0   \n",
       "4          poojan   NN      NN      NN    NN    NN      0             0   \n",
       "..            ...  ...     ...     ...   ...   ...    ...           ...   \n",
       "583  construction   NN      NN      NN    VB    VB      0             1   \n",
       "584          take   VB      NN      NN    CD    CD      0             1   \n",
       "585           six   CD      VB      NN   NNS   NNS      0             0   \n",
       "586        months  NNS      CD      VB    NN  none      0             0   \n",
       "587          year   NN     NNS      CD  none  none      0             0   \n",
       "\n",
       "     FativeVerb  FactiveVerbContext  ...  ReportVerbContext  StrongSub  \\\n",
       "0             0                   0  ...                  0          0   \n",
       "1             0                   0  ...                  0          0   \n",
       "2             0                   0  ...                  0          0   \n",
       "3             0                   0  ...                  0          0   \n",
       "4             0                   0  ...                  0          0   \n",
       "..          ...                 ...  ...                ...        ...   \n",
       "583           0                   0  ...                  1          0   \n",
       "584           0                   0  ...                  1          0   \n",
       "585           0                   0  ...                  0          0   \n",
       "586           0                   0  ...                  0          0   \n",
       "587           0                   0  ...                  0          0   \n",
       "\n",
       "     StrongSubContext  WeakSub  WeakSubContext  PositiveWord  \\\n",
       "0                   0        0               0             0   \n",
       "1                   0        0               0             0   \n",
       "2                   0        0               0             0   \n",
       "3                   0        0               0             0   \n",
       "4                   0        0               0             0   \n",
       "..                ...      ...             ...           ...   \n",
       "583                 0        0               1             0   \n",
       "584                 0        0               1             0   \n",
       "585                 0        0               0             0   \n",
       "586                 0        0               0             0   \n",
       "587                 0        0               0             0   \n",
       "\n",
       "     PositiveWordContext  NegativeWord  NegativeWordContext  BiasLexicon  \n",
       "0                      0             0                    0            0  \n",
       "1                      0             0                    0            0  \n",
       "2                      0             0                    0            0  \n",
       "3                      0             0                    0            0  \n",
       "4                      0             0                    0            0  \n",
       "..                   ...           ...                  ...          ...  \n",
       "583                    1             0                    0            0  \n",
       "584                    0             0                    0            1  \n",
       "585                    0             0                    0            0  \n",
       "586                    0             0                    0            0  \n",
       "587                    0             0                    0            1  \n",
       "\n",
       "[588 rows x 25 columns]"
      ]
     },
     "execution_count": 275,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Hedge': [0.3401360544217687], 'HedgeContext': [1.3605442176870748], 'FativeVerb': [0.6802721088435374], 'FactiveVerbContext': [2.7210884353741496], 'AssertiveVerb': [2.380952380952381], 'AssertiveVerbContext': [9.523809523809524], 'ImplicativeVerb': [0.3401360544217687], 'ImplicativeVerbContext': [1.3605442176870748], 'ReportVerb': [4.761904761904762], 'ReportVerbContext': [16.666666666666664], 'StrongSub': [2.0408163265306123], 'StrongSubContext': [7.8231292517006805], 'WeakSub': [7.8231292517006805], 'WeakSubContext': [26.190476190476193], 'PositiveWord': [2.380952380952381], 'PositiveWordContext': [9.183673469387756], 'NegativeWord': [1.0204081632653061], 'NegativeWordContext': [3.4013605442176873], 'BiasLexicon': [14.625850340136054]}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Hedge</th>\n",
       "      <th>HedgeContext</th>\n",
       "      <th>FativeVerb</th>\n",
       "      <th>FactiveVerbContext</th>\n",
       "      <th>AssertiveVerb</th>\n",
       "      <th>AssertiveVerbContext</th>\n",
       "      <th>ImplicativeVerb</th>\n",
       "      <th>ImplicativeVerbContext</th>\n",
       "      <th>ReportVerb</th>\n",
       "      <th>ReportVerbContext</th>\n",
       "      <th>StrongSub</th>\n",
       "      <th>StrongSubContext</th>\n",
       "      <th>WeakSub</th>\n",
       "      <th>WeakSubContext</th>\n",
       "      <th>PositiveWord</th>\n",
       "      <th>PositiveWordContext</th>\n",
       "      <th>NegativeWord</th>\n",
       "      <th>NegativeWordContext</th>\n",
       "      <th>BiasLexicon</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.340136</td>\n",
       "      <td>1.360544</td>\n",
       "      <td>0.680272</td>\n",
       "      <td>2.721088</td>\n",
       "      <td>2.380952</td>\n",
       "      <td>9.52381</td>\n",
       "      <td>0.340136</td>\n",
       "      <td>1.360544</td>\n",
       "      <td>4.761905</td>\n",
       "      <td>16.666667</td>\n",
       "      <td>2.040816</td>\n",
       "      <td>7.823129</td>\n",
       "      <td>7.823129</td>\n",
       "      <td>26.190476</td>\n",
       "      <td>2.380952</td>\n",
       "      <td>9.183673</td>\n",
       "      <td>1.020408</td>\n",
       "      <td>3.401361</td>\n",
       "      <td>14.62585</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Hedge  HedgeContext  FativeVerb  FactiveVerbContext  AssertiveVerb  \\\n",
       "0  0.340136      1.360544    0.680272            2.721088       2.380952   \n",
       "\n",
       "   AssertiveVerbContext  ImplicativeVerb  ImplicativeVerbContext  ReportVerb  \\\n",
       "0               9.52381         0.340136                1.360544    4.761905   \n",
       "\n",
       "   ReportVerbContext  StrongSub  StrongSubContext   WeakSub  WeakSubContext  \\\n",
       "0          16.666667   2.040816          7.823129  7.823129       26.190476   \n",
       "\n",
       "   PositiveWord  PositiveWordContext  NegativeWord  NegativeWordContext  \\\n",
       "0      2.380952             9.183673      1.020408             3.401361   \n",
       "\n",
       "   BiasLexicon  \n",
       "0     14.62585  "
      ]
     },
     "execution_count": 276,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''# some analysis\n",
    "result= {\n",
    "    'Property':[],\n",
    "    'Total':[],\n",
    "    'Percentage':[]\n",
    "}\n",
    "r,c = df.shape\n",
    "for col in df.columns:\n",
    "    if col!='word' and col!='POS' and col!='POSNeg1' and col!='POSNeg2' and col!='POS1' and col!='POS2':\n",
    "        #print(col ,\"            \", df[col].sum(),\"         \", df[col].sum()/r)\n",
    "        result['Property'].append(col)\n",
    "        result['Total'].append(df[col].sum())\n",
    "        result['Percentage'].append((df[col].sum()/r)*100)\n",
    "#print(result)\n",
    "'''\n",
    "r,c = df.shape\n",
    "result={\n",
    "        'Hedge':[],\n",
    "        'HedgeContext':[],\n",
    "        'FativeVerb':[],\n",
    "        'FactiveVerbContext':[],\n",
    "        'AssertiveVerb':[],\n",
    "        'AssertiveVerbContext':[],\n",
    "        'ImplicativeVerb':[],\n",
    "        'ImplicativeVerbContext':[],\n",
    "        'ReportVerb':[],\n",
    "        'ReportVerbContext':[],\n",
    "        'StrongSub':[],\n",
    "        'StrongSubContext':[],\n",
    "        'WeakSub':[],\n",
    "        'WeakSubContext':[],\n",
    "        'PositiveWord':[],\n",
    "        'PositiveWordContext':[],\n",
    "        'NegativeWord':[],\n",
    "        'NegativeWordContext':[],\n",
    "        'BiasLexicon':[]\n",
    "}\n",
    "for col in df.columns:\n",
    "    if col!='word' and col!='POS' and col!='POSNeg1' and col!='POSNeg2' and col!='POS1' and col!='POS2':\n",
    "        result[col].append((df[col].sum()/r)*100)\n",
    "print(result)\n",
    "\n",
    "df = pd.DataFrame(result, columns = [\n",
    "        'Hedge',\n",
    "        'HedgeContext',\n",
    "        'FativeVerb',\n",
    "        'FactiveVerbContext',\n",
    "        'AssertiveVerb',\n",
    "        'AssertiveVerbContext',\n",
    "        'ImplicativeVerb',\n",
    "        'ImplicativeVerbContext',\n",
    "        'ReportVerb',\n",
    "        'ReportVerbContext',\n",
    "        'StrongSub',\n",
    "        'StrongSubContext',\n",
    "        'WeakSub',\n",
    "        'WeakSubContext',\n",
    "        'PositiveWord',\n",
    "        'PositiveWordContext',\n",
    "        'NegativeWord',\n",
    "        'NegativeWordContext',\n",
    "        'BiasLexicon']\n",
    "        )\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 308,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df[['Hedge',\n",
    "        'HedgeContext',\n",
    "        'FativeVerb',\n",
    "        'FactiveVerbContext',\n",
    "        'AssertiveVerb',\n",
    "        'AssertiveVerbContext',\n",
    "        'ImplicativeVerb',\n",
    "        'ImplicativeVerbContext',\n",
    "        'ReportVerb',\n",
    "        'ReportVerbContext',\n",
    "        'StrongSub',\n",
    "        'StrongSubContext',\n",
    "        'WeakSub',\n",
    "        'WeakSubContext',\n",
    "        'PositiveWord',\n",
    "        'PositiveWordContext',\n",
    "        'NegativeWord',\n",
    "        'NegativeWordContext']]\n",
    "y = df['BiasLexicon']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 309,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.20,random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 310,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "logistic_regression= LogisticRegression()\n",
    "logistic_regression.fit(X_train,y_train)\n",
    "y_pred=logistic_regression.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "metadata": {},
   "outputs": [],
   "source": [
    "confusion_matrix = pd.crosstab(y_test, y_pred, rownames=['Actual'], colnames=['Predicted'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.7916666666666666\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ',metrics.accuracy_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix,classification_report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.79      0.98      0.88        55\n",
      "           1       0.75      0.18      0.29        17\n",
      "\n",
      "    accuracy                           0.79        72\n",
      "   macro avg       0.77      0.58      0.58        72\n",
      "weighted avg       0.78      0.79      0.74        72\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test,y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cutoff=[15,22,26,29,33]  these are perfect"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}