classify_sections_containing_relevant_information.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import re\n",
    "\n",
    "from path import Path, getcwdu\n",
    "\n",
    "import glob\n",
    "import os\n",
    "from pathlib import PurePath\n",
    "import copy\n",
    "from bs4 import BeautifulSoup as bs\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split, cross_val_score\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline \n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import classification_report, recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_file_list = [PurePath(file).name for file in glob.iglob('data/nc_training_filings/*')]\n",
    "train_path_list = [PurePath(os.getcwd()).joinpath(file).as_posix() for file in glob.iglob('data/nc_training_filings/*')]\n",
    "train_accession_ids = [PurePath(file).stem for file in train_file_list]\n",
    "train_cik_nbrs = [x.split(sep='-')[0] for x in train_accession_ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "val_accession_ids = train_accession_ids[300:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Helper function for viewing paragraph text in training dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_row_detail(df, nrow=10, header_list = ['ticker', 'accession_number' ],\n",
    "                    detail_list = ['data_key_friendly_name', 'text', 'paragraph_text'],\n",
    "                    sortby=['accession_number', 'data_key_friendly_name'], ascending=True):\n",
    "    df_sorted = df.sort_values(sortby, ascending=ascending).reset_index()\n",
    "    nrow = min(len(df_sorted), nrow)\n",
    "    for i in range(0, nrow):\n",
    "        for h in header_list:\n",
    "            print('-'*35  + ' ' +  str(df_sorted[h][i]) + ' ' + '-'*35)\n",
    "        for d in detail_list:\n",
    "            print(d + '  :' + str(df_sorted[d][i]))\n",
    "            print('')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functions for extracting candidate documents from an html file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function to create and index a dataframe from an html table. Empty cells are removed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def df_from_html_tbl(table_tag):\n",
    "    data_re = re.compile(r'[a-zA-Z0-9$().]+')\n",
    "    repl = re.compile(r'[()$]')\n",
    "    rows = []\n",
    "    for row in table_tag.findChildren('tr'):\n",
    "        row_list = []\n",
    "        for s in row.strings:\n",
    "            s = re.sub(repl, '', s.strip())\n",
    "            if len(s) > 0:\n",
    "                row_list.append(s)\n",
    "        if len(row_list) > 1:\n",
    "            rows.append(row_list)\n",
    "    tbl_df = pd.DataFrame.from_records(rows)\n",
    "    try:\n",
    "        tbl_df = tbl_df.set_index( tbl_df.applymap(len).max().values.argmax()) \n",
    "    except: tbl_df = tbl_df.set_index(0)\n",
    "    return tbl_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test and refine regex patterns for flagging likely relevant documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def check_regex_match(pattern, text_list):\n",
    "    for idx, s in enumerate(text_list):\n",
    "        mo = re.search(pattern, s)\n",
    "        if mo:\n",
    "            ms = mo.span()[1]\n",
    "            print(\"------    \" + str(idx) + \"   Matched!    -----\")\n",
    "            print('str length  :' + str(len(s)) + '    match span  :' + str(ms))\n",
    "            print(s[:ms])\n",
    "            print('')\n",
    "            print(s[ms:])\n",
    "            print(re.search(pattern, s))\n",
    "        else:\n",
    "            print(\"------    \" + str(idx) + \"  NO MATCH    -----\")\n",
    "            print(s)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Patterns used to filter after initial regex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nonum_pats = [r\"^[^\\d]*$\",\n",
    "    r\"^[^\\d]*\\d{1,2}[^\\d]*$\", \n",
    "    r\"^([^\\d]+(\\d{1,2}[^\\d]{2,}){1,5}\\d{1,3}[^\\d]*)$\",\n",
    "    r\"^((\\d{1,3}[^\\d]{2,}){1,4}\\d{1,2}[^\\d]*)$\"]\n",
    "year_and_num = re.compile(r\"20[0-2][0-9].*[0-9]{1,3}.*|[0-9]{1,3}.*20[0-2][0-9]\", re.I)\n",
    "nonum_regs = [re.compile(x) for x in nonum_pats]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "paragraph_input_df = pd.read_csv('data/paragraph_input_df.csv', index_col=0)\n",
    "paragraph_input_df['split'] = paragraph_input_df['split'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 4259 entries, 0 to 4258\n",
      "Data columns (total 5 columns):\n",
      "acc_id       4259 non-null object\n",
      "para_text    4259 non-null object\n",
      "len          4259 non-null int64\n",
      "split        4259 non-null category\n",
      "label        4259 non-null int64\n",
      "dtypes: category(1), int64(2), object(2)\n",
      "memory usage: 170.6+ KB\n"
     ]
    }
   ],
   "source": [
    "paragraph_input_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acc_id</th>\n",
       "      <th>para_text</th>\n",
       "      <th>len</th>\n",
       "      <th>split</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0001193125-17-319357</td>\n",
       "      <td>During 1998, we announced a program permitting...</td>\n",
       "      <td>943</td>\n",
       "      <td>train</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0001193125-17-319357</td>\n",
       "      <td>Subsequent to August 26, 2017, we have repurch...</td>\n",
       "      <td>288</td>\n",
       "      <td>train</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 acc_id                                          para_text  \\\n",
       "0  0001193125-17-319357  During 1998, we announced a program permitting...   \n",
       "1  0001193125-17-319357  Subsequent to August 26, 2017, we have repurch...   \n",
       "\n",
       "   len  split  label  \n",
       "0  943  train      1  \n",
       "1  288  train      1  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paragraph_input_df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# After reading back in, the html needs to be parsed again\n",
    "tbl_html_df = pd.read_csv('tbl_html_df.csv')\n",
    "tbl_html_df['tbl_html'] = tbl_html_df.tbl_html.apply(lambda x: bs(x, 'lxml'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assign documents to positive and negative lists based on regex; also clean text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pos_list = paragraph_input_df.para_text[paragraph_input_df.label == 1].values.tolist()\n",
    "neg_list = paragraph_input_df.para_text[paragraph_input_df.label == 0].values.tolist()\n",
    "pos_key_list = paragraph_input_df.acc_id[paragraph_input_df.label == 1].values.tolist()\n",
    "neg_key_list = paragraph_input_df.acc_id[paragraph_input_df.label == 0].values.tolist()\n",
    "\n",
    "pos_list_train = [pos_list[i] for i,v in enumerate(pos_key_list) if v not in val_accession_ids]\n",
    "pos_list_val = [pos_list[i] for i,v in enumerate(pos_key_list) if v in val_accession_ids]\n",
    "neg_list_train = [neg_list[i] for i,v in enumerate(neg_key_list) if v not in val_accession_ids]\n",
    "neg_list_val = [neg_list[i] for i,v in enumerate(neg_key_list) if v in val_accession_ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pos_tup_train = [(pos_key_list[i], pos_list[i]) for i,v in enumerate(pos_key_list) if v not in val_accession_ids]\n",
    "pos_tup_val = [(pos_key_list[i], pos_list[i]) for i,v in enumerate(pos_key_list) if v in val_accession_ids]\n",
    "neg_tup_train = [(neg_key_list[i], neg_list[i]) for i,v in enumerate(neg_key_list) if v not in val_accession_ids]\n",
    "neg_tup_val = [(neg_key_list[i], neg_list[i]) for i,v in enumerate(neg_key_list) if v  in val_accession_ids]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "neg_key_list_train, neg_list_train = ([k[0] for k in neg_tup_train], [k[1] for k in neg_tup_train])\n",
    "neg_key_list_val, neg_list_val = ([k[0] for k in neg_tup_val], [k[1] for k in neg_tup_val])\n",
    "pos_key_list_train, pos_list_train = ([k[0] for k in pos_tup_train], [k[1] for k in pos_tup_train])\n",
    "pos_key_list_val, pos_list_val = ([k[0] for k in pos_tup_val], [k[1] for k in pos_tup_val])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "54 keys keys missed out of 300 covered  (11 if table texts are included) (2 only had table paragraphs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total paragraphs with positive hits: 1098\n",
      "training paragraphs with positive hits: 764\n",
      "training keys with positive hits: 246 out of 300\n",
      "validation keys with positive hits: 112 out of 135\n"
     ]
    }
   ],
   "source": [
    "print('total paragraphs with positive hits: ' + str(len(pos_list)))\n",
    "print('training paragraphs with positive hits: ' + str(len(pos_list_train)))\n",
    "print('training keys with positive hits: ' + str(len(set(pos_key_list_train))) + ' out of 300')\n",
    "print('validation keys with positive hits: ' + str(len(set(pos_key_list_val))) + ' out of 135')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total paragraphs with no regex match: 3161\n",
      "training paragraphs with no regex match: 2240\n",
      "training keys with paragraphs labeled negative: 294 out of 300\n",
      "validation keys with paragraphs labeled negative: 133 out of 135\n"
     ]
    }
   ],
   "source": [
    "print('total paragraphs with no regex match: ' + str(len(neg_list)))\n",
    "print('training paragraphs with no regex match: ' + str(len(neg_list_train)))\n",
    "print('training keys with paragraphs labeled negative: ' + str(len(set(neg_key_list_train))) + ' out of 300')\n",
    "print('validation keys with paragraphs labeled negative: ' + str(len(set(neg_key_list_val))) + ' out of 135')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "List of keys with no paragraphs flagged as relevant by the regex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "missed_keys = [key for key in neg_key_list if key not in pos_key_list ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Used for manually labeling documents as relevant or not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_docs_from_list(key_list: list=neg_key_list, doc_list: list=neg_list, start: int=0, ndocs: int=50):\n",
    "    end = start + ndocs\n",
    "    for idx, tup in enumerate(zip(key_list[start:end], doc_list[start:end])):\n",
    "        print(str(idx + start) + '   ------   ' + str(tup[0]))\n",
    "        print(tup[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "false_pos_indices = []\n",
    "false_neg_indices = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# After reading back in, the html needs to be parsed again\n",
    "tbl_html_df = pd.read_csv('data/tbl_html_df.csv')\n",
    "tbl_html_df['tbl_html'] = tbl_html_df.tbl_html.apply(lambda x: bs(x, 'lxml'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "false_pos = []; false_neg = [];\n",
    "false_pos = [pos_list[:200][i] for i in false_pos_indices  ]\n",
    "false_neg = [neg_list[:200][i] for i in false_neg_indices  ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pos_labeled = [x for x in pos_list[:200] if x not in false_pos] + false_neg\n",
    "neg_labeled = [x for x in neg_list[:200] if x not in false_neg] + false_pos\n",
    "train_labeled = pos_labeled + neg_labeled"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building labeled training set for first document classifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find the tokens that best identify misses from the regex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "\n",
    "from nltk.tokenize import sent_tokenize, PunktSentenceTokenizer\n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tokenize dollar values as dollar tokens, numerics as numeric tokens, but leave years as they are"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "year_pat = re.compile(r\"([^0-9])((?:20|19)[0,1,2,9][0-9])([^0-9])\")\n",
    "dollar_pat = re.compile(r\"[$]([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{,4}\")\n",
    "num_pat = re.compile(\n",
    "r\"(y(?:20|19)[0,1,2,9][0-9])|(?:(?P<bound1>[\\s,.])(?:(?:[0-9]{1,3}[,])*(?:[0-9]{1,3}))(?:[.][0-9]{1,4})?[%]?(?P<bound2>[\\s,.]))\")\n",
    "year_fix_pat = re.compile(r\"y((?:20|19)[0,1,2,9][0-9])num_tok\")\n",
    "\n",
    "def replace_numeric_toks(s):\n",
    "    s1 = re.sub(year_pat, r'\\1y\\2\\3', s )\n",
    "    s2 = re.sub(dollar_pat, r'dollar_tok', s1)\n",
    "    s3 = re.sub(num_pat, r'\\1\\g<bound1>num_tok\\g<bound2>', s2)\n",
    "    s4 = re.sub(year_fix_pat, r'\\1', s3)\n",
    "    return s4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train_labeled_tok_ready = [replace_numeric_toks(x) for x in train_labeled]\n",
    "#y = np.array(list(np.repeat(1, len(pos_labeled))) + list(np.repeat(0, len(neg_labeled))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = [replace_numeric_toks(x) for x in pos_list + neg_list]\n",
    "X_train = [replace_numeric_toks(x) for x in pos_list_train + neg_list_train]\n",
    "X_val = [replace_numeric_toks(x) for x in pos_list_val + neg_list_val]\n",
    "y = np.array(list(np.repeat(1, len(pos_list))) + list(np.repeat(0, len(neg_list))))\n",
    "y_train = np.array(list(np.repeat(1, len(pos_list_train))) + list(np.repeat(0, len(neg_list_train))))\n",
    "y_val = np.array(list(np.repeat(1, len(pos_list_val))) + list(np.repeat(0, len(neg_list_val))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function to cross-validate models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cv_acc(model, X, y, cv=5, scoring='accuracy'):\n",
    "    cv_dict = {}\n",
    "    cvs = cross_val_score(model, X, y, cv=cv, scoring=scoring)\n",
    "    cv_dict['cv_mean'] = np.mean(cvs)\n",
    "    cv_dict['cvs'] = cvs\n",
    "    return cv_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_params_dict = {}\n",
    "model_cvs = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "count_vec = CountVectorizer(ngram_range=(1,6), max_df=0.6, min_df=.025, max_features=5000)\n",
    "bin_vec = CountVectorizer(ngram_range=(1,6), max_df=0.2, min_df=.01, binary=True, max_features=5000)\n",
    "tfidf_vec = TfidfVectorizer(ngram_range=(1,6), max_df=0.62, min_df=.025, max_features=5000)\n",
    "\n",
    "# Define a pipeline combining a text feature extractor with a simple\n",
    "# classifier\n",
    "\n",
    "# Logistic Regression \n",
    "lr_tfidf_pl = Pipeline([\n",
    "    ('vec', tfidf_vec),\n",
    "    ('lr', LogisticRegression(random_state=14, max_iter=1000))\n",
    "])\n",
    "\n",
    "lr_bin_pl = Pipeline([\n",
    "    ('vec', bin_vec),\n",
    "    ('lr', LogisticRegression(random_state=14, max_iter=1000))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "param_grid_bin = {\n",
    "    'vec__max_df': (0.15, .2, .25),\n",
    "    'vec__min_df': (.005, .01, .015),\n",
    "    'vec__ngram_range' : [(1,6)]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "param_grid_tfidf = {\n",
    "    'vec__max_df': (.4, 0.5, .6, .7,),\n",
    "    'vec__min_df': (.005, .01, .015, .025),\n",
    "    'vec__ngram_range' : [(1,6)]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr_tfidf_gs = GridSearchCV(lr_tfidf_pl, param_grid=param_grid_tfidf, cv=4, return_train_score=True)\n",
    "lr_bin_gs = GridSearchCV(lr_bin_pl, param_grid=param_grid_bin, cv=4, return_train_score=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=4, error_score='raise',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('vec', CountVectorizer(analyzer='word', binary=True, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=0.2, max_features=5000, min_df=0.01,\n",
       "        ngram_range=(1, 6), preprocessor=None, stop_words=None,\n",
       "        stri...alty='l2', random_state=14, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))]),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'vec__max_df': (0.15, 0.2, 0.25), 'vec__min_df': (0.005, 0.01, 0.015), 'vec__ngram_range': [(1, 6)]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_bin_gs.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=4, error_score='raise',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('vec', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=0.62, max_features=5000, min_df=0.025,\n",
       "        ngram_range=(1, 6), norm='l2', preprocessor=None, smooth_idf=True...alty='l2', random_state=14, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))]),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'vec__max_df': (0.4, 0.5, 0.6, 0.7), 'vec__min_df': (0.005, 0.01, 0.015, 0.025), 'vec__ngram_range': [(1, 6), (1, 7)]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score=True,\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_tfidf_gs.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'vec__max_df': 0.15, 'vec__min_df': 0.005, 'vec__ngram_range': (1, 6)}"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Best hyperparameter settings, which were found by the GridSearchCV - want to keep this:\n",
    "model_params_dict['lr_bin'] = lr_bin_gs.best_params_\n",
    "lr_bin_gs.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'vec__max_df': 0.7, 'vec__min_df': 0.01, 'vec__ngram_range': (1, 6)}"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Best hyperparameter settings, which were found by the GridSearchCV - want to keep this:\n",
    "model_params_dict['lr_tfidf'] = lr_tfidf_gs.best_params_\n",
    "lr_tfidf_gs.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8868175765645806"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_tfidf_gs.cv_results_['mean_test_score'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9171105193075899"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_bin_gs.cv_results_['mean_test_score'].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Manually entering the parameters already discovered via GridsearchCV\n",
    "model_params_dict['lr_bin'] = {'vec__max_df': 0.15, 'vec__min_df': 0.005, 'vec__ngram_range': (1, 6)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set the model params using the dictionary. The `**` unpacks the dictionary into the key-value pairs - essentially, it removes the curly braces."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('vec', CountVectorizer(analyzer='word', binary=True, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=0.15, max_features=5000, min_df=0.005,\n",
       "        ngram_range=(1, 6), preprocessor=None, stop_words=None,\n",
       "        st...alty='l2', random_state=14, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_bin_pl.set_params(**model_params_dict['lr_bin'])\n",
    "lr_bin_pl.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('vec', TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=0.7, max_features=5000, min_df=0.01,\n",
       "        ngram_range=(1, 6), norm='l2', preprocessor=None, smooth_idf=True,\n",
       "...alty='l2', random_state=14, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False))])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_tfidf_pl.set_params(**model_params_dict['lr_tfidf'])\n",
    "lr_tfidf_pl.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get and print model cross validation scores:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'cv_mean': 0.9187692734331669, 'cvs': array([0.92678869, 0.90682196, 0.93178037, 0.92678869, 0.90166667])}\n"
     ]
    }
   ],
   "source": [
    "model_cvs['lr_bin'] = cv_acc(lr_bin_pl, X_train, y_train)\n",
    "print(model_cvs['lr_bin'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Against the unseen holdout/validation set: Basically the same as the CV folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9051792828685259"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_bin_pl.score(X_val, y_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'cv_mean': 0.8888053244592345, 'cvs': array([0.91181364, 0.86189684, 0.90682196, 0.90349418, 0.86      ])}\n"
     ]
    }
   ],
   "source": [
    "model_cvs['lr_tfidf'] = cv_acc(lr_tfidf_pl, X_train, y_train)\n",
    "print(model_cvs['lr_tfidf'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_pred = lr_bin_pl.predict(X)\n",
    "y_val_pred = lr_bin_pl.predict(X_val)\n",
    "y_proba = lr_bin_pl.predict_proba(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "misses = y_pred != y\n",
    "#X_test_misses = [X_test[i] for i in range(0, len(misses)) if misses[i] == True]\n",
    "X_misses = [X[i] for i in range(0, len(misses)) if misses[i] == True]\n",
    "len(X_misses)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Second element in each row has the probability of y=1, according to the LR model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7600372060407212"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_proba[:,1].tolist()[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------    0    --------------------\n",
      "True value :  1\n",
      "Predicted probability :  0.8921753830869105\n",
      "Snap-on has undertaken repurchases of Snap-on common stock from time to time to offset dilution created by shares issued for employee and franchisee stock purchase plans, stock awards and other corporate purposes. Snap-on repurchased num_tok shares, num_tok shares and num_tok shares in 2017, 2016 and 2015, respectively. As of 2017 year end, Snap-on has remaining availability to repurchase up to an additional dollar_tok million in common stock pursuant to Board authorizations. The purchase of Snap-on common stock is at the company's discretion, subject to prevailing financial and market conditions.\n",
      "--------------------    1    --------------------\n",
      "True value :  1\n",
      "Predicted probability :  0.10782461691308949\n",
      "On November num_tok, 2016, we announced plans to purchase up to dollar_tok billion of our common stock through 2019. On March num_tok, 2017, we announced plans to double our share repurchase program to dollar_tok billion of common stock through 2019, with dollar_tok billion allocated and purchased in 2017, and the remainder allocated evenly to 2018 and 2019. On February num_tok, 2018, we announced the acceleration of our previously stated 2018 share repurchases from dollar_tok billion to dollar_tok billion, with the remaining balance to be repurchased in 2019. Acquisitions for the share repurchase program are made at management's discretion, at prevailing prices, subject to market conditions and other factors. Repurchases may be increased, decreased or discontinued at any time without prior notice. Shares of stock repurchased under the plan are held as treasury shares.\n"
     ]
    }
   ],
   "source": [
    "for idx, tup in enumerate(zip(X_misses, y_proba[misses][1].tolist(), y[misses])):\n",
    "    print(\"--------------------    \" + str(idx) + \"    --------------------\" )\n",
    "    print(\"Labeled value :  \" + str(tup[2]))\n",
    "    print(\"Predicted probability :  \" + str(tup[1]))\n",
    "    print(tup[0])\n",
    "    if idx > 9:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.92      0.96      0.94       921\n",
      "          1       0.86      0.76      0.81       334\n",
      "\n",
      "avg / total       0.90      0.91      0.90      1255\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_val, y_val_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coef_tfidf_df = pd.DataFrame.from_records(list(zip(lr_tfidf_pl.named_steps.vec.get_feature_names(), \n",
    "                                             lr_tfidf_pl.named_steps.lr.coef_.tolist()[0])),\n",
    "                                   columns = ['tfidf_token', 'tfidf_coef']).sort_values('tfidf_coef', ascending=False).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "coef_bin_df = pd.DataFrame.from_records(list(zip(lr_bin_pl.named_steps.vec.get_feature_names(), \n",
    "                                             lr_bin_pl.named_steps.lr.coef_.tolist()[0])),\n",
    "                                   columns = ['token', 'coef']).sort_values('coef', ascending=False).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tfidf_token</th>\n",
       "      <th>tfidf_coef</th>\n",
       "      <th>token</th>\n",
       "      <th>coef</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>repurchased num_tok</td>\n",
       "      <td>3.032666</td>\n",
       "      <td>accelerated share</td>\n",
       "      <td>2.183093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>accelerated share</td>\n",
       "      <td>2.971670</td>\n",
       "      <td>accelerated share repurchase</td>\n",
       "      <td>1.975529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>accelerated share repurchase</td>\n",
       "      <td>2.772415</td>\n",
       "      <td>repurchased num_tok</td>\n",
       "      <td>1.936068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>accelerated</td>\n",
       "      <td>2.380541</td>\n",
       "      <td>authorized share repurchase</td>\n",
       "      <td>1.421375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>repurchased num_tok million</td>\n",
       "      <td>2.228513</td>\n",
       "      <td>repurchased num_tok million shares</td>\n",
       "      <td>1.242539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>repurchased num_tok million shares</td>\n",
       "      <td>2.047673</td>\n",
       "      <td>accelerated</td>\n",
       "      <td>1.171833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>num_tok million shares</td>\n",
       "      <td>1.948307</td>\n",
       "      <td>repurchased num_tok shares</td>\n",
       "      <td>1.131840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>million shares</td>\n",
       "      <td>1.946904</td>\n",
       "      <td>repurchased num_tok million</td>\n",
       "      <td>1.125678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>we repurchased num_tok</td>\n",
       "      <td>1.751198</td>\n",
       "      <td>authorized share</td>\n",
       "      <td>1.096774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>authorized share repurchase</td>\n",
       "      <td>1.744254</td>\n",
       "      <td>authorized the repurchase</td>\n",
       "      <td>1.053928</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>authorized the repurchase</td>\n",
       "      <td>1.694291</td>\n",
       "      <td>authorized the repurchase of</td>\n",
       "      <td>0.971496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>authorized the repurchase of</td>\n",
       "      <td>1.677915</td>\n",
       "      <td>authorized share repurchase program</td>\n",
       "      <td>0.915360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>repurchased num_tok shares</td>\n",
       "      <td>1.644017</td>\n",
       "      <td>num_tok million shares of</td>\n",
       "      <td>0.840565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>num_tok shares</td>\n",
       "      <td>1.641122</td>\n",
       "      <td>num_tok shares of</td>\n",
       "      <td>0.812177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>common stock</td>\n",
       "      <td>1.630167</td>\n",
       "      <td>remained available</td>\n",
       "      <td>0.797729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>authorized share</td>\n",
       "      <td>1.553978</td>\n",
       "      <td>million shares of</td>\n",
       "      <td>0.772944</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>repurchase</td>\n",
       "      <td>1.549792</td>\n",
       "      <td>accelerated share repurchase program</td>\n",
       "      <td>0.730288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>shares</td>\n",
       "      <td>1.511762</td>\n",
       "      <td>remained</td>\n",
       "      <td>0.683435</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>million</td>\n",
       "      <td>1.476729</td>\n",
       "      <td>num_tok shares of our common</td>\n",
       "      <td>0.682931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>our common stock</td>\n",
       "      <td>1.457117</td>\n",
       "      <td>num_tok shares of our</td>\n",
       "      <td>0.667155</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           tfidf_token  tfidf_coef  \\\n",
       "0                  repurchased num_tok    3.032666   \n",
       "1                    accelerated share    2.971670   \n",
       "2         accelerated share repurchase    2.772415   \n",
       "3                          accelerated    2.380541   \n",
       "4          repurchased num_tok million    2.228513   \n",
       "5   repurchased num_tok million shares    2.047673   \n",
       "6               num_tok million shares    1.948307   \n",
       "7                       million shares    1.946904   \n",
       "8               we repurchased num_tok    1.751198   \n",
       "9          authorized share repurchase    1.744254   \n",
       "10           authorized the repurchase    1.694291   \n",
       "11        authorized the repurchase of    1.677915   \n",
       "12          repurchased num_tok shares    1.644017   \n",
       "13                      num_tok shares    1.641122   \n",
       "14                        common stock    1.630167   \n",
       "15                    authorized share    1.553978   \n",
       "16                          repurchase    1.549792   \n",
       "17                              shares    1.511762   \n",
       "18                             million    1.476729   \n",
       "19                    our common stock    1.457117   \n",
       "\n",
       "                                   token      coef  \n",
       "0                      accelerated share  2.183093  \n",
       "1           accelerated share repurchase  1.975529  \n",
       "2                    repurchased num_tok  1.936068  \n",
       "3            authorized share repurchase  1.421375  \n",
       "4     repurchased num_tok million shares  1.242539  \n",
       "5                            accelerated  1.171833  \n",
       "6             repurchased num_tok shares  1.131840  \n",
       "7            repurchased num_tok million  1.125678  \n",
       "8                       authorized share  1.096774  \n",
       "9              authorized the repurchase  1.053928  \n",
       "10          authorized the repurchase of  0.971496  \n",
       "11   authorized share repurchase program  0.915360  \n",
       "12             num_tok million shares of  0.840565  \n",
       "13                     num_tok shares of  0.812177  \n",
       "14                    remained available  0.797729  \n",
       "15                     million shares of  0.772944  \n",
       "16  accelerated share repurchase program  0.730288  \n",
       "17                              remained  0.683435  \n",
       "18          num_tok shares of our common  0.682931  \n",
       "19                 num_tok shares of our  0.667155  "
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([coef_tfidf_df.head(20), coef_bin_df.head(20)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tfidf_token</th>\n",
       "      <th>tfidf_coef</th>\n",
       "      <th>token</th>\n",
       "      <th>coef</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>issued</td>\n",
       "      <td>-0.621430</td>\n",
       "      <td>over</td>\n",
       "      <td>-0.388198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>following</td>\n",
       "      <td>-0.623518</td>\n",
       "      <td>at the</td>\n",
       "      <td>-0.389273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>income</td>\n",
       "      <td>-0.639322</td>\n",
       "      <td>transaction</td>\n",
       "      <td>-0.404682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>on the</td>\n",
       "      <td>-0.650178</td>\n",
       "      <td>the following</td>\n",
       "      <td>-0.407363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>november num_tok 2017</td>\n",
       "      <td>-0.654391</td>\n",
       "      <td>discretion</td>\n",
       "      <td>-0.409595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>and dollar_tok</td>\n",
       "      <td>-0.666065</td>\n",
       "      <td>november num_tok 2017</td>\n",
       "      <td>-0.434913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>table</td>\n",
       "      <td>-0.677647</td>\n",
       "      <td>million of common</td>\n",
       "      <td>-0.441840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>by the</td>\n",
       "      <td>-0.679719</td>\n",
       "      <td>dollar_tok million of common</td>\n",
       "      <td>-0.444438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>million remaining</td>\n",
       "      <td>-0.680569</td>\n",
       "      <td>2017 and</td>\n",
       "      <td>-0.455317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>capacity</td>\n",
       "      <td>-0.699376</td>\n",
       "      <td>of shares</td>\n",
       "      <td>-0.470325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>net</td>\n",
       "      <td>-0.712502</td>\n",
       "      <td>agreement</td>\n",
       "      <td>-0.483528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>securities</td>\n",
       "      <td>-0.713246</td>\n",
       "      <td>securities</td>\n",
       "      <td>-0.511833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>the following</td>\n",
       "      <td>-0.740039</td>\n",
       "      <td>num_tok million of</td>\n",
       "      <td>-0.512285</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>repurchased and</td>\n",
       "      <td>-0.749711</td>\n",
       "      <td>include</td>\n",
       "      <td>-0.525782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>or</td>\n",
       "      <td>-0.817464</td>\n",
       "      <td>in 2017 we</td>\n",
       "      <td>-0.539617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>the dollar_tok</td>\n",
       "      <td>-0.855998</td>\n",
       "      <td>by the</td>\n",
       "      <td>-0.550916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>is</td>\n",
       "      <td>-0.938921</td>\n",
       "      <td>in december</td>\n",
       "      <td>-0.555714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>notes</td>\n",
       "      <td>-1.139032</td>\n",
       "      <td>repurchased an</td>\n",
       "      <td>-0.569496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>of the</td>\n",
       "      <td>-1.145827</td>\n",
       "      <td>the dollar_tok</td>\n",
       "      <td>-0.627538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>by</td>\n",
       "      <td>-1.321953</td>\n",
       "      <td>capacity</td>\n",
       "      <td>-0.670564</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              tfidf_token  tfidf_coef                         token      coef\n",
       "0                  issued   -0.621430                          over -0.388198\n",
       "1               following   -0.623518                        at the -0.389273\n",
       "2                  income   -0.639322                   transaction -0.404682\n",
       "3                  on the   -0.650178                 the following -0.407363\n",
       "4   november num_tok 2017   -0.654391                    discretion -0.409595\n",
       "5          and dollar_tok   -0.666065         november num_tok 2017 -0.434913\n",
       "6                   table   -0.677647             million of common -0.441840\n",
       "7                  by the   -0.679719  dollar_tok million of common -0.444438\n",
       "8       million remaining   -0.680569                      2017 and -0.455317\n",
       "9                capacity   -0.699376                     of shares -0.470325\n",
       "10                    net   -0.712502                     agreement -0.483528\n",
       "11             securities   -0.713246                    securities -0.511833\n",
       "12          the following   -0.740039            num_tok million of -0.512285\n",
       "13        repurchased and   -0.749711                       include -0.525782\n",
       "14                     or   -0.817464                    in 2017 we -0.539617\n",
       "15         the dollar_tok   -0.855998                        by the -0.550916\n",
       "16                     is   -0.938921                   in december -0.555714\n",
       "17                  notes   -1.139032                repurchased an -0.569496\n",
       "18                 of the   -1.145827                the dollar_tok -0.627538\n",
       "19                     by   -1.321953                      capacity -0.670564"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([coef_tfidf_df.tail(20).reset_index(drop=True), coef_bin_df.tail(20).reset_index(drop=True)], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "paragraph_input_df['rel_proba'] = y_proba[:,1].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acc_id</th>\n",
       "      <th>para_text</th>\n",
       "      <th>len</th>\n",
       "      <th>split</th>\n",
       "      <th>label</th>\n",
       "      <th>rel_proba</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0000002969-17-000039</td>\n",
       "      <td>On 15 September 2011, the Board of Directors a...</td>\n",
       "      <td>654</td>\n",
       "      <td>train</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0000002969-17-000039</td>\n",
       "      <td>On 15 September 2011, the Board of Directors a...</td>\n",
       "      <td>282</td>\n",
       "      <td>train</td>\n",
       "      <td>1</td>\n",
       "      <td>0.998451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0000002969-17-000039</td>\n",
       "      <td>On 15 September 2011, the Board of Directors a...</td>\n",
       "      <td>442</td>\n",
       "      <td>train</td>\n",
       "      <td>1</td>\n",
       "      <td>0.997926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0000004127-17-000033</td>\n",
       "      <td>(1)The share repurchase program approved by th...</td>\n",
       "      <td>369</td>\n",
       "      <td>train</td>\n",
       "      <td>1</td>\n",
       "      <td>0.996196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0000004127-17-000033</td>\n",
       "      <td>(3) 600,000 shares were repurchased at an aver...</td>\n",
       "      <td>235</td>\n",
       "      <td>train</td>\n",
       "      <td>1</td>\n",
       "      <td>0.920757</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 acc_id                                          para_text  \\\n",
       "0  0000002969-17-000039  On 15 September 2011, the Board of Directors a...   \n",
       "1  0000002969-17-000039  On 15 September 2011, the Board of Directors a...   \n",
       "2  0000002969-17-000039  On 15 September 2011, the Board of Directors a...   \n",
       "3  0000004127-17-000033  (1)The share repurchase program approved by th...   \n",
       "4  0000004127-17-000033  (3) 600,000 shares were repurchased at an aver...   \n",
       "\n",
       "   len  split  label  rel_proba  \n",
       "0  654  train      1   0.999732  \n",
       "1  282  train      1   0.998451  \n",
       "2  442  train      1   0.997926  \n",
       "3  369  train      1   0.996196  \n",
       "4  235  train      1   0.920757  "
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paragraph_input_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenize into sentences and classify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "15 September 2011           1\n",
       "1986                        1\n",
       "1999                        1\n",
       "2001                        1\n",
       "2004                        1\n",
       "2009                        1\n",
       "2012                        1\n",
       "2013                        2\n",
       "2015                        1\n",
       "2016                        3\n",
       "2017                        2\n",
       "April 2016                  1\n",
       "April 2017                  7\n",
       "April num_tok, 2014         1\n",
       "April num_tok, 2015         4\n",
       "April num_tok, 2016         4\n",
       "April num_tok, 2017         4\n",
       "August 2004                 2\n",
       "August 2015                 2\n",
       "August 2016                 2\n",
       "August 2017                 3\n",
       "August num_tok, 2000        1\n",
       "August num_tok, 2015        2\n",
       "August num_tok, 2016        2\n",
       "August num_tok, 2017        3\n",
       "December 2013               2\n",
       "December 2014               2\n",
       "December 2015               1\n",
       "December 2016               6\n",
       "December 2017              14\n",
       "                           ..\n",
       "November 2016               8\n",
       "November 2017               7\n",
       "November num_tok, 2012      1\n",
       "November num_tok, 2014      4\n",
       "November num_tok, 2015      2\n",
       "November num_tok, 2016      6\n",
       "November num_tok, 2017     13\n",
       "November, 2017              1\n",
       "October 2005                1\n",
       "October 2010                1\n",
       "October 2015                1\n",
       "October 2016                3\n",
       "October 2017                5\n",
       "October num_tok, 2014       2\n",
       "October num_tok, 2015       2\n",
       "October num_tok, 2016       2\n",
       "October num_tok, 2017       7\n",
       "September 2001              3\n",
       "September 2014              1\n",
       "September 2015              2\n",
       "September 2016              2\n",
       "September 2017              2\n",
       "September num_tok, 2007     1\n",
       "September num_tok, 2013     1\n",
       "September num_tok, 2015     2\n",
       "September num_tok, 2016     6\n",
       "September num_tok, 2017     6\n",
       "first quarter of 2016       2\n",
       "fiscal 2014                 1\n",
       "second quarter of 2017      2\n",
       "Name: text, Length: 132, dtype: int64"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "srp_df[srp_df.data_key_friendly_name == 'Share Repurchase Authorization Date'].text.apply(replace_numeric_toks).value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "auth_date_pat_list = [\n",
    "r\"(jan\\w+|feb\\w+|march|april|may|june|july|aug\\w+|sept\\w+|octob\\w+|nov\\w+|decem\\w+) (20)?[0-3]?[0-9],( 20[1-2][0-9][,])?\",\n",
    "#r\"[0-3][0-9][,] (jan\\w+|feb\\w+|march|april|may|june|july|aug\\w+|sept\\w+|nov\\w+|decem\\w+)( 20[1-2][0-9])?\",\n",
    "r\"(20|19)[9,0,1][0-9][,]?((\\w+|[()-,])* )*((announc\\w*|author\\w*|approv\\w*) )((\\w+|[()-,])* )*repurchase ((\\w+|[()-,])* )*((our|common|outstanding) (shar(e|es)|stock))\" \n",
    "    #r\"((we|the|\\w+[']s) ){1,2}(board( of directors)? )?(of (the company|\\w+([']s))? )?(approved|announced|authorized) (((a|the) (new )?)|(an increase to the (current )?(authoriz\\w+ )?(for the )?))((stock|common stock|standing share) )?((rep|p)urchase) (authorization|plan|program)?(,)? (by|for|authoriz\\w+|whereby the Company may repurchase) (up to )?([$])?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{,4})( (m|b)illion )?\", \n",
    "]\n",
    "\n",
    "auth_date_regs = [re.compile(x, re.I) for x in auth_date_pat_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "para_list = paragraph_input_df.para_text.values.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def month_tok(s):\n",
    "    \"\"\"Replace month names with month_tok. \"\"\"\n",
    "    month_pat = re.compile(r\"jan\\w+|feb\\w+|march|april|june|july|aug\\w+|sept\\w+|octob\\w+|nov\\w+|decem\\w+\", re.I)\n",
    "    may_pat = re.compile(r\"\\WM(ay|AY)\\W\")\n",
    "    s_tok = re.sub(month_pat, \"month_tok\", s)\n",
    "    s_tok = re.sub(may_pat, \"month_tok\", s_tok)\n",
    "    return s_tok\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "_sre.SRE_Pattern"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(auth_date_regs[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%time len([any(reg.search(x) for reg in auth_date_regs) for x in  para_list[:100]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%time len(x if any(reg.search(x))  for x in  para_list[:1] for reg in auth_date_regs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "434"
      ]
     },
     "execution_count": 220,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "len(paragraph_input_df.groupby('acc_id').first())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "paragraph_input_df['auth_date_hit'] = paragraph_input_df.para_text.apply(lambda x: any(reg.search(x) for reg in auth_date_regs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "reg_bs = paragraph_input_df.para_text.apply(lambda x: any(reg.search(x) for reg in auth_date_regs))\n",
    "len(paragraph_input_df[reg_bs].groupby('acc_id').first())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Labels vs. model output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------- 0000002969-17-000039 -----------------------------------\n",
      "label  :1\n",
      "\n",
      "rel_proba  :0.9979259632276545\n",
      "\n",
      "para_text  :On 15 September 2011, the Board of Directors authorized the repurchase of up to $1,000 of our outstanding common stock. We repurchase shares pursuant to Rules 10b5-1 and 10b-18 under the Securities Exchange Act of 1934, as amended, through repurchase agreements established with several brokers. We did not purchase any of our outstanding shares during fiscal year 2017. At 30 September 2017, $485.3 in share repurchase authorization remains.\n",
      "\n",
      "----------------------------------- 0000002969-17-000039 -----------------------------------\n",
      "label  :1\n",
      "\n",
      "rel_proba  :0.998451479822388\n",
      "\n",
      "para_text  :On 15 September 2011, the Board of Directors authorized the repurchase of up to $1,000 of our outstanding common stock. We did not purchase any of our outstanding shares during fiscal years 2017, 2016 or 2015 . At 30 September 2017, $485.3 in share repurchase authorization remains.\n",
      "\n",
      "----------------------------------- 0000002969-17-000039 -----------------------------------\n",
      "label  :1\n",
      "\n",
      "rel_proba  :0.9997318222103452\n",
      "\n",
      "para_text  :On 15 September 2011, the Board of Directors authorized the repurchase of up to $1.0 billion of our outstanding common stock. This program does not have a stated expiration date. We repurchase shares pursuant to Rules 10b5-1 and 10b-18 under the Securities Exchange Act of 1934, as amended, through repurchase agreements established with several brokers. There were no purchases of stock during fiscal year 2017 . At 30 September 2017, $485.3 million in share repurchase authorization remained. Additional purchases will be completed at the Company's discretion while maintaining sufficient funds for investing in its businesses and growth opportunities.\n",
      "\n",
      "----------------------------------- 0000004127-17-000033 -----------------------------------\n",
      "label  :0\n",
      "\n",
      "rel_proba  :0.0010289380845300623\n",
      "\n",
      "para_text  :Employee Stock Purchase Plans. The Company maintains a domestic and an international employee stock purchase plan. Under these plans, eligible employees may purchase common stock through payroll deductions of up to 10% of their compensation. The price per share is the lower of 85% of the fair market value of the common stock at the beginning or end of each offering period (generally six months). The plans provide for purchases by employees of up to an aggregate of 9.7 million shares. Shares of common stock purchased under these plans in the fiscal years ended September 29, 2017, September 30, 2016, and October 2, 2015, were 0.2 million, 0.3 million, and 0.3 million, respectively. At September 29, 2017, there are 0.7 million shares available for purchase. The Company recognized compensation expense of $4.5 million, $4.6 million and $4.7 million for the fiscal years ended September 29, 2017, September 30, 2016, and October 2, 2015, respectively, related to the employee stock purchase plan. The unrecognized compensation expense on the employee stock purchase plan at September 29, 2017, was $1.8 million . The weighted average period over which the cost is expected to be recognized is approximately four months.\n",
      "\n",
      "----------------------------------- 0000004127-17-000033 -----------------------------------\n",
      "label  :0\n",
      "\n",
      "rel_proba  :0.001987399750524349\n",
      "\n",
      "para_text  :The number of stockholders of record of our common stock as of November 3, 2017 was 14,389. On November 6, 2017, the Board of Directors declared a cash dividend of $0.32 per share of common stock, payable on December 12, 2017, to stockholders of record as of November 21, 2017. We intend to continue to pay quarterly dividends subject to capital availability and our view that cash dividends are in the best interests of our stockholders. Future cash dividends may be affected by, among other items, our views on potential future capital requirements, including those relating to research and development, creation and expansion of sales distribution channels and investments and acquisitions, legal risks, stock repurchase programs, debt issuance, changes in federal and state income tax law and changes to our business model.\n",
      "\n",
      "----------------------------------- 0000004127-17-000033 -----------------------------------\n",
      "label  :0\n",
      "\n",
      "rel_proba  :0.003868529984099636\n",
      "\n",
      "para_text  :The following table provides information regarding repurchases of common stock made during the fiscal quarter ended September 29, 2017 :\n",
      "\n",
      "----------------------------------- 0000004127-17-000033 -----------------------------------\n",
      "label  :0\n",
      "\n",
      "rel_proba  :0.008672148148316005\n",
      "\n",
      "para_text  :Cash and cash equivalent balances were $1,616.8 million at September 29, 2017, representing an increase of $533.0 million from September 30, 2016 . The increase resulted from $1,471.3 million in cash generated from operations which was partially offset by $432.3 million used to repurchase 4.7 million shares of stock, and $214.6 million in cash dividend payments during fiscal 2017, $303.3 million in capital expenditures and $13.7 million related to business acquisition activity. Based on our historical results of operations, we expect that our cash and cash equivalents on hand and the cash we expect to generate from operations will be sufficient to fund our research and development, capital expenditures, potential acquisitions, working capital, quarterly cash dividend payments (if such dividends are declared by the Board of Directors), outstanding commitments and other liquidity requirements associated with existing operations for at least the next 12 months. However, we cannot be certain that our cash on hand and cash generated from operations will be available in the future to fund all of our capital and operating requirements. In addition, any future strategic investments and acquisitions may require additional cash and capital resources. If we are unable to obtain sufficient cash or capital to meet our needs on a timely basis and on favorable terms, our business and operations could be materially and adversely affected.\n",
      "\n",
      "----------------------------------- 0000004127-17-000033 -----------------------------------\n",
      "label  :0\n",
      "\n",
      "rel_proba  :0.009281074174932217\n",
      "\n",
      "para_text  :Cash flow used in financing activities consists primarily of cash transactions related to debt and equity. During fiscal 2017, we had net cash outflows of $612.4 million, compared to $804.6 million in fiscal 2016. The decrease in cash used in financing activities primarily related to the decrease in share repurchase activity primarily offset by increased dividend payments during fiscal 2017. During fiscal 2017 we had the following significant uses of cash:\n",
      "\n",
      "----------------------------------- 0000004127-17-000033 -----------------------------------\n",
      "label  :1\n",
      "\n",
      "rel_proba  :0.9207574566605661\n",
      "\n",
      "para_text  :(3) 600,000 shares were repurchased at an average price of $102.28 per share as part of our share repurchase program and 2,137 shares were withheld for tax obligations under restricted stock agreements with an average price of $105.53.\n",
      "\n",
      "----------------------------------- 0000004127-17-000033 -----------------------------------\n",
      "label  :1\n",
      "\n",
      "rel_proba  :0.9207574566605661\n",
      "\n",
      "para_text  :(4) 400,000 shares were repurchased at an average price of $101.08 per share as part of our share repurchase program and 2,615 shares were withheld for tax obligations under restricted stock agreements with an average price of $101.54.\n",
      "\n",
      "----------------------------------- 0000004127-17-000033 -----------------------------------\n",
      "label  :1\n",
      "\n",
      "rel_proba  :0.9664992703831703\n",
      "\n",
      "para_text  :During the fiscal year ended September 30, 2016, the Company paid approximately $525.6 million (including commissions) in connection with the repurchase of 8.0 million shares of its common stock (paying an average price of $65.70 per share).\n",
      "\n",
      "----------------------------------- 0000004127-17-000033 -----------------------------------\n",
      "label  :1\n",
      "\n",
      "rel_proba  :0.9961959627794287\n",
      "\n",
      "para_text  :(1)The share repurchase program approved by the Board of Directors on January 17, 2017, authorized the repurchase of up to $500.0 million of our common stock from time to time on the open market or in privately negotiated transactions as permitted by securities laws and other legal requirements. The share repurchase program is scheduled to expire on January 17, 2019.\n",
      "\n",
      "----------------------------------- 0000004127-17-000033 -----------------------------------\n",
      "label  :1\n",
      "\n",
      "rel_proba  :0.9999996942266509\n",
      "\n",
      "para_text  :On January 17, 2017, the Board of Directors approved a new share repurchase program, pursuant to which the Company is authorized to repurchase up to $500.0 million of its common stock from time to time prior to January 17, 2019, on the open market or in privately negotiated transactions as permitted by securities laws and other legal requirements. This authorized share repurchase program replaced in its entirety the July 19, 2016, share repurchase program. During the fiscal year ended September 29, 2017, the Company paid approximately $432.3 million (including commissions) in connection with the repurchase of 4.7 million shares of its common stock (paying an average price of $92.97 per share) under the January 17, 2017, share repurchase plan and the July 19, 2016, share repurchase plan. As of September 29, 2017, $174.1 million remained available under the January 17, 2017, share repurchase plan.\n",
      "\n",
      "----------------------------------- 0000004281-18-000042 -----------------------------------\n",
      "label  :1\n",
      "\n",
      "rel_proba  :0.9936193461733234\n",
      "\n",
      "para_text  :To further enhance the Company's financial position and return capital to shareholders, Arconic's Board of Directors authorized a share repurchase program of up to $500 of its outstanding common stock and a $500 early debt reduction. Under the share repurchase program, the Company may repurchase shares from time to time, in amounts, at prices, and at such times as the Company deems appropriate. Repurchases will be subject to market conditions, legal requirements and other considerations. The Company is not obligated to repurchase any specific number of shares or to do so at any particular time, and the share repurchase program may be suspended, modified or terminated at any time without prior notice. For the early debt reduction, Arconic intends to redeem in March 2018 all of its outstanding 5.72% Notes due in 2019.\n",
      "\n",
      "----------------------------------- 0000004281-18-000042 -----------------------------------\n",
      "label  :1\n",
      "\n",
      "rel_proba  :0.9936927209582255\n",
      "\n",
      "para_text  :In February 2018, the Company announced that its Board of Directors authorized a share repurchase program of up to $500 of its outstanding common stock and a $500 early debt reduction. Under the share repurchase program, the Company may repurchase shares from time to time, in amounts, at prices, and at such times as the Company deems appropriate. Repurchases will be subject to market conditions, legal requirements and other considerations. Arconic is not obligated to repurchase any specific number of shares or to do so at any particular time, and the share repurchase program may be suspended, modified or terminated at any time without prior notice. For the early debt reduction, Arconic intends to redeem in March 2018 all of its outstanding 5.72% Notes due in 2019.\n",
      "\n",
      "----------------------------------- 0000004962-18-000032 -----------------------------------\n",
      "label  :0\n",
      "\n",
      "rel_proba  :1.5578305195753464e-05\n",
      "\n",
      "para_text  :The Company is limited in its ability to pay dividends by the Federal Reserve, which could prohibit a dividend that would be considered an unsafe or unsound banking practice. It is the policy of the Federal Reserve that bank holding companies generally should pay dividends on preferred and common stock only out of net income available to common shareholders generated over the past year, and only if prospective earnings retention is consistent with the organization's current and expected future capital needs, asset quality and overall financial condition. Moreover, bank holding companies are required by statute to be a source of strength to their insured depository institution subsidiaries and should not maintain dividend levels that undermine their ability to do so. On an annual basis, the Company is required to develop and maintain a capital plan, which includes planned dividends over a two-year horizon. The Company may be limited in its ability to pay dividends if the Federal Reserve objects to its capital plan. In addition, the Capital Rules include a capital conservation buffer which is being phased in from January 1, 2016 through January 1, 2019. The Capital Rules also include a countercyclical capital buffer, which is currently set at zero but which could be increased by the Federal Reserve in the future. These buffers can be satisfied only with CET1 capital. If the Company's risk-based capital ratios were to fall below the applicable buffer levels, the Company would be subject to certain restrictions on dividends, stock repurchases and other capital distributions, as well as discretionary bonus payments to executive officers.\n",
      "\n",
      "----------------------------------- 0000004962-18-000032 -----------------------------------\n",
      "label  :0\n",
      "\n",
      "rel_proba  :1.9368355748269342e-05\n",
      "\n",
      "para_text  :Common shares are generally retired by the Company upon repurchase (except for 2.9 million, 3.0 million and 3.0 million shares held as treasury shares as of December 31, 2017, 2016 and 2015, respectively); retired common shares and treasury shares are excluded from the shares outstanding in the table above. The treasury shares, with a cost basis of $217 million, $197 million and $242 million as of December 31, 2017, 2016 and 2015, respectively, are included as a reduction to additional paid-in capital in shareholders' equity on the Consolidated Balance Sheets.\n",
      "\n",
      "----------------------------------- 0000004962-18-000032 -----------------------------------\n",
      "label  :0\n",
      "\n",
      "rel_proba  :0.0005327509004136939\n",
      "\n",
      "para_text  :In the fourth quarter of 2017, we recognized a tax charge of $2.6 billion related to the Tax Act, which drove a decline in net income versus 2016. This charge represents our current estimate of taxes on deemed repatriations of certain overseas earnings and the remeasurement of U.S. net deferred tax assets. Our effective tax rate for 2017 was up substantially from 33 percent in 2016. Excluding the impacts of the Tax Act, our effective tax rate for the year would have decreased compared to 2016, primarily due to the realization of certain foreign tax credits in the current year and a continuing shift in the geographic mix of earnings. We continue to analyze and interpret the Tax Act, and its impact on our earnings; however, for 2018, we currently estimate our tax rate will be approximately 22 percent, before discrete tax items. The upfront charge triggered by the Tax Act reduced our capital ratios and, as a result, while we will be continuing our quarterly dividends at the current level, we suspended our share buyback program for the first half of 2018 in order to rebuild our capital.\n",
      "\n",
      "----------------------------------- 0000004962-18-000032 -----------------------------------\n",
      "label  :0\n",
      "\n",
      "rel_proba  :0.006203075128088531\n",
      "\n",
      "para_text  :As previously mentioned, we decided to suspend our share buyback program for the first half of 2018 in order to rebuild our capital levels and ratios. We intend to continue our quarterly dividends during the first half of 2018 at the current level. Authorization for share repurchases beginning in the second half of 2018 must be submitted as part of our capital plan within the CCAR 2018 process.\n",
      "\n",
      "----------------------------------- 0000004962-18-000032 -----------------------------------\n",
      "label  :0\n",
      "\n",
      "rel_proba  :0.011411168391894119\n",
      "\n",
      "para_text  :Due to the Tax Act impact of $2.6 billion, we reported a net loss in the fourth quarter. The net loss, combined with growth in the balance sheet and continued capital return in the quarter, resulted in a decline in our Common Equity Tier 1 Risk-Based Capital ratio to 9.0 percent, which is below the level we had projected in the 2017 CCAR process. As a result, we have suspended our share repurchase program for the first half of 2018 in order to rebuild our capital levels.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_row_detail(df=paragraph_input_df, nrow=20, header_list = ['acc_id'],\n",
    "                    detail_list = ['label', 'rel_proba', 'para_text'],\n",
    "                    sortby=['acc_id', 'rel_proba'], ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sent_pat_list = [r\"Board( of Directors)?( has)? authorized the repurchase\", \n",
    "r\"(on|in|at|as of) (jan\\w+|feb\\w+|march|april|may|june|july|august|sept\\w+|novem\\w+|decem\\w+) (20)?[0-9]{1,2}, (20[1-2][0-9][,] )?((we|the|\\w+[']s) ){1,2}(board( of directors)? )?(of (the company|\\w+([']s))? )?(approved|announced|authorized) (((a|the) (new )?)|(an increase to the (current )?(authoriz\\w+ )?(for the )?))((stock|common stock|standing share) )?((rep|p)urchase) (authorization|plan|program)?(,)? (by|for|authoriz\\w+|whereby the Company may repurchase) (up to )?([$])?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{,4})( (m|b)illion )?\", \n",
    "r\"((we|the|\\w+[']s) ){1,2}(board( of directors)? )?(of (the company|\\w+([']s))? )?(approved|announced|authorized) (((a|the) (new )?)|(an increase to the (current )?(authoriz\\w+ )?(for the )?))((stock|common stock|standing share) )?((rep|p)urchase) (authorization|plan|program)?(,)? (by|for|authoriz\\w+|whereby the Company may repurchase) (up to )?([$])?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{,4})( (m|b)illion)?\",       \n",
    "r\"Board( of Directors)?( has)? (approved|authorized) a( new)? (share|stock) repurchase program\", \n",
    "r\"Board( of Directors)? approved an( additional)? increase in the stock repurchase\",\n",
    "r\"share repurchase authorization by the board\",\n",
    "r\"shares (rep|p)urchased as Part of Public\", \n",
    "r\"accelerated share repurchase\",     \n",
    "r\"(re)?purchase(d)?[,]? (up to )?(an aggregate |in aggregate, |a )?(total )?(of )?((up to|approximately) )?(([$])?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{0,4}) ((m|b)illion )?(shares )?)(and (([$])?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{0,4}) ((m|b)illion )?(shares )?))?(shares )?of ((our|its|the|/w+[']s) ){1,2}common stock\",\n",
    "\n",
    "r\"(authorized|approved) a share repurchase program\", \n",
    "r\"authorized the repurchase of (shares|up to)\", \n",
    "r\"authorized repurchases of up to ([$])?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{,4}) ((m|b)illion )shares\",\n",
    "r\"authorized share repurchase program\", \n",
    "r\"authorization replace(d|s) (the|all|any) prior repurchase authorization\", \n",
    "r\"([$])?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{,4})( (m|b)illion)? shares were repurchased\", \n",
    "r\"(approximately )?[$](([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{0,4})( (m|b)illion)? to repurchase (approximately )?[$]?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{,4}) ((m|b)illion )?(shares )?of (\\w+([']s)? )?Common Stock\",\n",
    "r\"repurchased ([$])?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{,4}) (million )?(common )?shares\", \n",
    "r\"([$])?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{,4}) ((m|b)illion )? stock repurchase\",\n",
    "r\"(Company|we) (repurchased|purchased) (approximately )?([$])?([0-9]{,4}[.]?[0-9]{,4}) million shares\", \n",
    "r\"we did not repurchase any shares\", \n",
    "\n",
    "r\"([$])?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{0,4}) ((m|b)illion )?(shares )?(of our common stock )?remain((s|ing|ed) )?under ((the|our|publicly announced) ){0,3}((authoriz/w+|program(s)?|share|repurchase)\\s?){1,4}\",\n",
    "r\"([$])?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{0,4}) ((m|b)illion )?(shares )?(of our common stock )?remain((s|ing|ed) )?to be repurchase\",\n",
    "r\"million common shares remaining under the\", \n",
    "r\"remai(n|ned|ning) under (our|the) share repurchase (authorization|program)\", \n",
    "r\"The Company currently plans to (rep|p)urchase ([$]?)([0-9]{,4}[.]?[0-9]{,4}) ((m|b)illion )?(to ([$]?)([0-9]{,4}[.]?[0-9]{,4}) (m|b)illion )?(shares|of its common stock)\",\n",
    "r\"(approximately )?([$]?)([0-9]{1,4}[.]?[0-9]{0,4}) (m|b)illion ((shares|of|our|common|stock) ){0,5}remai(ned|n) (available|under the authorization|authorized)\",\n",
    "r\"ha(d|s) (approximately )([$]?)([0-9]{1,4}[.]?[0-9]{0,4}) (m|b)illion remaining under ((the|this )?repurchase authorization\",\n",
    "        \n",
    "           ]\n",
    "\n",
    "r\"(shares|amount) ((available|remaining) )((for|under|the|share) ){1,5}repurchase ((\\w+)[,]? ){0,4}share repurchase((\\w+)[,]? ){1,6}([$])?(([0-9]{1,3},)*[0-9]{1,3}[.]?[0-9]{0,4})( (m|b)illion)?( share(s)?)?\",            \n",
    "r\"share repurchase authorization remain|remained|by the board)\",\n",
    "r\"Shares (remaining )?that May Yet Be Purchased\", \n",
    "remain_reg_list = [re.compile(x, re.I) for x in pat_list]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}