merge_preproc_2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "from bs4 import BeautifulSoup as bs\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process(chk,i):\n",
    "    pat = re.compile(r'\\d+\\s+U\\.S.\\s+\\d+', re.MULTILINE|re.UNICODE)\n",
    "    val = []\n",
    "    chk = chk.drop_duplicates(subset='html',keep='first')\n",
    "    not_nan = chk[pd.notnull(chk['html'])]\n",
    "    for i,c in enumerate(not_nan['html']): \n",
    "        try:                                                     \n",
    "            patmatch = re.findall(pat,c)[0]\n",
    "            val.append(patmatch)\n",
    "        except: \n",
    "            val.append(None)\n",
    "            \n",
    "    not_nan['US_CITE'] = val\n",
    "    not_nan = not_nan[pd.notnull(not_nan['US_CITE'])]\n",
    "\n",
    "    gs = pd.read_csv('case_meta_data.csv', encoding = \"ISO-8859-1\")\n",
    "    gs['US_CITE'] = gs['usCite']\n",
    "    #print(gs.head(1))\n",
    "    #print(\"\")\n",
    "    #print(not_nan.head(1))\n",
    "    \n",
    "    result = pd.merge(not_nan,gs,how='left',on='US_CITE')\n",
    "    #print(result.head(1))\n",
    "\n",
    "    result = result.drop(['Unnamed: 0','absolute_url','author','author_str','cluster','date_created','date_modified','date_created','date_modified','download_url','extracted_by_ocr','html_columbia','html_lawbox','html_with_citations','joined_by','local_path','opinions_cited','page_count','per_curiam','plain_text','resource_uri','sha1','type','caseId','usCite'], axis=1)\n",
    "    \n",
    "    html_cleaner = lambda x: bs(x).text\n",
    "    result['html'] = result['html'].apply(html_cleaner)\n",
    "    # column you are working on\n",
    "    html_df = result['html']\n",
    "    stopword_set = set(stopwords.words(\"english\"))\n",
    "    # convert to lower case and split \n",
    "    html_df = html_df.str.lower()\n",
    "    #print(html_df.head(2))\n",
    "    splitter = lambda x: x.split()\n",
    "    html_df.apply(splitter)\n",
    "    #print(html_df.head(2))\n",
    "    # remove stopwords\n",
    "    html_df_list = html_df.tolist()\n",
    "    html_df_wo_stops = [w for w in html_df_list if not w in stopwords.words(\"english\")]\n",
    "    #print(html_df_list[:2])\n",
    "    # keep only words\n",
    "    html_only_words = [re.sub(r'[^a-zA-Z\\s]','', i) for i in html_df_wo_stops]\n",
    "    #print(html_only_words[:2])\n",
    "    # join the cleaned words in a list\n",
    "    html_df = pd.DataFrame({\"processed_html\": html_only_words})\n",
    "    html_df.replace(\"\\n\", \" \").replace('\\r', '')\n",
    "    result['html'] = html_df\n",
    "\n",
    "    result.to_csv(str(i)+'-proc.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/lib/python3.6/site-packages/ipykernel_launcher.py:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  del sys.path[0]\n",
      "/anaconda/lib/python3.6/site-packages/bs4/__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file /anaconda/lib/python3.6/runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP})\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP, \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    }
   ],
   "source": [
    "chunk_size = 10**4\n",
    "i = 0\n",
    "\n",
    "for chunk in pd.read_csv('scotus_file',chunksize=chunk_size):\n",
    "    process(chunk, i)\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "gs.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "result.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"9471-proc.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "len(df['US_CITE'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "len((df['html'][pd.isnull(df['US_CITE'])]).unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "sum(df.US_CITE.isnull())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = df.drop(df.columns[[0,1,2,3,4,5,6,7,8,9,11,12,13,14,15,16,17,18,19,20,21,22,24,25,26]], axis=1)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bs\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "html_cleaner = lambda x: bs(x).text\n",
    "df['html'] = df['html'].apply(html_cleaner)\n",
    "\n",
    "# column you are working on\n",
    "html_df = df['html'].head(50)\n",
    "\n",
    "stopword_set = set(stopwords.words(\"english\"))\n",
    "\n",
    "# convert to lower case and split \n",
    "html_df = html_df.str.lower()\n",
    "splitter = lambda x: x.split()\n",
    "html_df.apply(splitter)\n",
    "\n",
    "# remove stopwords\n",
    "html_df_list = html_df.tolist()\n",
    "html_df_wo_stops = [w for w in html_df_list if not w in stopwords.words(\"english\")]\n",
    "\n",
    "# keep only words\n",
    "html_only_words = [re.sub(r'[^a-zA-Z]',' ', i) for i in html_df_wo_stops]\n",
    "\n",
    "# join the cleaned words in a list\n",
    "html_df = pd.DataFrame({\"processed_html\": html_only_words})\n",
    "html_df.replace(\"\\n\", \" \").replace('\\r', '')\n",
    "\n",
    "df['html'] = html_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df1 = df.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df.append(df1, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}