PHI augmentation-your-own-dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If users already have de-identification datasets in BIO format, users can directly conduct PHI Augmentation by modifying the following code.\n",
    "\n",
    "Please check whether the PHI types in your own dataset are included in the provided named-entity lists in 'all_PHI_from_internet' directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "file_path_entity = 'C:/Users/zhou/Desktop/all_PHI_from_internet/'\n",
    "labels_in_2006 = ['PATIENT', 'DOCTOR', 'HOSPITAL', 'IDNUM', 'DATE', 'PHONE', 'AGE', 'ZIP',\n",
    "                  'MEDICALRECORD', 'ORGANIZATION', 'PROFESSION', 'USERNAME', 'LOCATION']\n",
    "\n",
    "label_content = [[] for i in range(len(labels_in_2006))]\n",
    "for k in range(len(labels_in_2006)):\n",
    "    file_path = file_path_entity + labels_in_2006[k] + '.txt'\n",
    "    with open(file_path, 'r', encoding='utf-8') as f_w:\n",
    "        content = f_w.readlines()\n",
    "        for j in content:\n",
    "            label_content[k].append(j.strip())\n",
    "\n",
    "\n",
    "def replace_PHI(label, line_number):\n",
    "    preserve_list = []\n",
    "    if label in labels_in_2006:\n",
    "        label_index = labels_in_2006.index(label)\n",
    "        words_list = label_content[label_index]\n",
    "\n",
    "        random.seed(global_seed)  # every epoch, it produces different dataset, also reimplementable.\n",
    "        random.shuffle(words_list)\n",
    "        words_num = len(words_list)\n",
    "\n",
    "        random.seed(line_number)  # line_number is defined for specific PHI instance\n",
    "        random_index = random.randint(0, words_num - 1)\n",
    "        random_w = words_list[random_index].strip()  # words_list is different at different epoch\n",
    "\n",
    "        words_num = random_w.split()\n",
    "        if len(words_num) == 1:\n",
    "            new_line = random_w+'   '+'B-'+label\n",
    "            preserve_list.append(new_line)\n",
    "        if len(words_num) > 1:\n",
    "            for i in range(len(words_num)):\n",
    "                if i == 0:\n",
    "                    new_line = words_num[i]+'   '+'B-'+label\n",
    "                    preserve_list.append(new_line)\n",
    "                if i > 0:\n",
    "                    new_line = words_num[i]+'   '+'I-'+label\n",
    "                    preserve_list.append(new_line)\n",
    "    return preserve_list\n",
    "\n",
    "\n",
    "file_path_2014 = 'C:/Users/zhou/Desktop/processed_2014_train.txt'\n",
    "global_seed = 2\n",
    "repalced_file_path = 'C:/Users/zhou/Desktop/new_replaced_2014_train_seed'+str(global_seed)+'.txt'\n",
    "\n",
    "\n",
    "reserve_content = []\n",
    "with open(file_path_2014, 'r', encoding='utf-8') as f_wr:\n",
    "    content = f_wr.readlines()\n",
    "    for row in range(0, len(content)-1):\n",
    "        if len(content[row]) == 1:  # cope with empty line\n",
    "            reserve_content.append('')\n",
    "        if len(content[row]) != 1 and len(content[row+1]) != 1:\n",
    "            all_token_Next = content[row + 1].split()\n",
    "            all_token = content[row].split()\n",
    "            this_line_label = all_token[-1]\n",
    "            next_line_label = all_token_Next[-1]\n",
    "            if this_line_label.startswith('O'):\n",
    "                new_line = all_token[0]+'   '+all_token[-1]\n",
    "                reserve_content.append(new_line)\n",
    "                continue\n",
    "            if this_line_label.startswith('B-') and next_line_label.startswith('I-'):\n",
    "                continue\n",
    "            if this_line_label.startswith('B-') and not next_line_label.startswith('I-'):\n",
    "                label = this_line_label.split('-')[-1]\n",
    "                new_words = replace_PHI(label, row)\n",
    "                for k in new_words:\n",
    "                    reserve_content.append(k)\n",
    "            if this_line_label.startswith('I-') and next_line_label.startswith('I-'):\n",
    "                continue\n",
    "            if this_line_label.startswith('I-') and not next_line_label.startswith('I-'):\n",
    "                label = this_line_label.split('-')[-1]\n",
    "                # print(label)\n",
    "                new_words = replace_PHI(label, row)\n",
    "                # print(new_words)\n",
    "                for k in new_words:\n",
    "                    reserve_content.append(k)\n",
    "\n",
    "        if len(content[row]) != 1 and len(content[row+1]) == 1:\n",
    "            all_token = content[row].split()\n",
    "            this_line_label = all_token[-1]\n",
    "            if this_line_label.startswith('B-') or this_line_label.startswith('I-'):\n",
    "                label = this_line_label.split('-')[-1]\n",
    "                new_words = replace_PHI(label, row)\n",
    "                for k in new_words:\n",
    "                    reserve_content.append(k)\n",
    "            if this_line_label.startswith('O'):\n",
    "                new_line = all_token[0]+'   '+all_token[-1]\n",
    "                reserve_content.append(new_line)\n",
    "                continue\n",
    "\n",
    "\n",
    "with open(repalced_file_path, 'w', encoding='utf-8') as f_w:\n",
    "    for line in reserve_content:\n",
    "        f_w.writelines(line+'\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}