report_preprocessor.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Rx thorax report preprocessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Unnamed: 0  codigoinforme                             v\n",
      "0           0        2442821             Sin alteraciones.\n",
      "1           1        2442822             Sin alteraciones.\n",
      "2           2        2442820            Sin  alteraciones.\n",
      "3           3        2442823  Signos radiológicos de EPOC.\n",
      "4           4        2442824             Sin alteraciones.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "df = pd.read_csv('report_sentences.csv', encoding=\"ISO-8859-1\")\n",
    "print(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convert text column to lowercase\n",
    "df['v'] = df['v'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#import unidecode to remove accents\n",
    "import unidecode\n",
    "df['v'] = df.apply(lambda row: unidecode.unidecode(str(row['v'])), axis=1 )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The field clinical evaluation \"v\", not only contains the Rx thorax description and clinical judgment but it may also include the reason for the Rx order as well as the descriptions of other Rx types such as sinus and abdominal radiographies in a non structured format. The following regex approachs are intended to extract the Rx thorax description and clinical judgments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "105809\n"
     ]
    }
   ],
   "source": [
    "#Extract \"Impresión Diagnóstica(*)\" in new column\n",
    "p = re.compile(\"Impresion\\s+diagnostica(.*)\", re.IGNORECASE)\n",
    "df[\"v_clean\"] = df.apply(lambda row: p.search(str(row['v'])).group(1) if p.search(str(row['v'])) else '', axis=1)\n",
    "p = re.compile(\"Impresion\\s+\\w+\\s?:(.*)\", re.IGNORECASE)\n",
    "df[\"v_clean\"] = df.apply(lambda row: p.search(str(row['v'])).group(1) if (row['v_clean']=='' and p.search(str(row['v']))) else row['v_clean'], axis=1)\n",
    "print(len(df[(df['v_clean']!= '')]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "109253\n"
     ]
    }
   ],
   "source": [
    "#Otherwise add \"torax:(*):\" to this column\n",
    "p = re.compile(\"torax.*?:(.*?)[a-zA-Z]*:\", re.IGNORECASE)\n",
    "df[\"v_clean\"] = df.apply(lambda row: p.search(str(row['v'])).group(1) if (row['v_clean']=='' and p.search(str(row['v']))) else row['v_clean'], axis=1)\n",
    "print(len(df[(df['v_clean']!= '')]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "184671\n"
     ]
    }
   ],
   "source": [
    "#Otherwise add \"torax:(*)\" to this column\n",
    "p = re.compile(\"torax.*?:(.*)\", re.IGNORECASE)\n",
    "df[\"v_clean\"] = df.apply(lambda row: p.search(str(row['v'])).group(1) if (row['v_clean']=='' and p.search(str(row['v']))) else row['v_clean'], axis=1)\n",
    "print(len(df[(df['v_clean']!= '')]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "184673\n"
     ]
    }
   ],
   "source": [
    "#Otherwise add \"Motivo de consulta:*Comentario:(*)\" to this column\n",
    "p = re.compile(\"motivo de consulta.*?comentario(.*)\", re.IGNORECASE)\n",
    "df[\"v_clean\"] = df.apply(lambda row: p.search(str(row['v'])).group(1) if (row['v_clean']=='' and p.search(str(row['v']))) else row['v_clean'], axis=1)\n",
    "print(len(df[(df['v_clean']!= '')]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "189361\n"
     ]
    }
   ],
   "source": [
    "#Otherwise add \":(*)\" to this column\n",
    "p = re.compile(\":(.*)\", re.IGNORECASE)\n",
    "df[\"v_clean\"] = df.apply(lambda row: p.search(str(row['v'])).group(1) if (row['v_clean']=='' and p.search(str(row['v']))) else row['v_clean'], axis=1)\n",
    "print(len(df[(df['v_clean']!= '')]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "206369\n"
     ]
    }
   ],
   "source": [
    "#Otherwise add \"(*)\" to this column\n",
    "df[\"v_clean\"] = df.apply(lambda row: str(row['v']) if row['v_clean']==''  else row['v_clean'], axis=1)\n",
    "print(len(df[(df['v_clean']!= '')]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv(\"report_sentences_selected.csv\", columns=['codigoinforme','v_clean'], encoding=\"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply common preprocessor pipeline: Normalization, tokenization, stopword and stemming (lemmatization and POS tagging does not work in spanish using NLTK, but those tasks are not necessary for the present problem) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('report_sentences_selected.csv', encoding=\"ISO-8859-1\", na_filter=False)\n",
    "# Remove punctuation characters except '.' as this define sentences\n",
    "df['v_clean'] = df.apply(lambda row: re.sub(r\"[^a-zA-Z0-9\\\\.]\", \" \", row['v_clean']), axis=1 )\n",
    "df['v_clean'] = df.apply(lambda row: row['v_clean'].replace('.', ' . '), axis=1)\n",
    "#df['v_clean'] = df.apply(lambda row: re.sub('(.*?)\\\\.\\s*(.*)', r'\\1 . \\2', row['v_clean']), axis=1 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.to_csv(\"report_sentences_cleaned.csv\", columns=['codigoinforme','v_clean'], encoding=\"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Stemmize and remove stopwords\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.snowball import SpanishStemmer\n",
    "stopwords = set(stopwords.words(\"spanish\")) - set(['sin', 'no', 'ni', 'con'])\n",
    "stemmer = SpanishStemmer()\n",
    "\n",
    "reports = []\n",
    "\n",
    "for report in pd.Series(df['v_clean']).tolist():\n",
    "    new_report = ''\n",
    "    for w in report.split():\n",
    "        w = w.strip()\n",
    "        if w not in stopwords:\n",
    "            if w not in set(['masa','masas']): #Avoid 'masa' and 'masas' to be stemmed to 'mas'\n",
    "                new_report = new_report + ' ' + stemmer.stem(w)\n",
    "            else:\n",
    "                new_report = new_report + ' ' + w\n",
    "    reports.append(new_report)\n",
    "df['v_preprocessed'] = pd.Series(reports)   \n",
    "df.to_csv(\"report_sentences_preprocessed.csv\", columns=['codigoinforme','v_preprocessed'], encoding=\"ISO-8859-1\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fo = open(\"text.txt\", \"w\")\n",
    "fo.write(df['v_clean'].str.cat(sep=' ').replace('.', ''))\n",
    "fo.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df."
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}