Text Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tracer\n",
    "\n",
    "1. The first column contains unique sentence IDs.\n",
    "2. The second column contains the sentence.\n",
    "3. The third column can contain the date of the file creation in the YYYY-MM-DD format or the word NULL. If you use NULL, make sure it is written in upper-case.\n",
    "4. The fourth column contains the book or section from which the sentence is taken. This information is crucial for the visualisation shown in the Text Reuse Browser figure in Postprocessing. The top drop-down menu you see there will list the information you provide in this fourth column.\n",
    "\n",
    "https://gfranzini.gitbooks.io/tracer/content/manual/corpus-preparation.html "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tracer_format(filename, title_prefix):\n",
    "    cnt = 0\n",
    "    with open(filename, \"r\", encoding=\"utf-8\") as f:\n",
    "        sents = f.read().split(\"\\n\")\n",
    "    with open(filename.replace(\".txt\", \"_tracer.txt\"), \"w\", encoding=\"utf-8\") as f1:\n",
    "        for sent in sents:\n",
    "            cnt+=1\n",
    "            c = str(cnt)\n",
    "            while len(c) < 6:\n",
    "                c = '0' + c\n",
    "            sentid = title_prefix + c\n",
    "            f1.write(\"%s\\t%s\\tNULL\\t%s\\n\" % (sentid, sent, filename[:-4]))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "prfs = [\"11\", \"12\", \"13\", \"14\"]\n",
    "titles = [\"cath_almaine_1.txt\", \"cath_almaine_2.txt\", \"cath_almaine_3.txt\", \"cath_almaine_diplomatic.txt\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for t, p in zip(titles, prfs):\n",
    "    tracer_format(t, p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def passim_format(filename):\n",
    "    title = filename[:-4]\n",
    "    with open(filename, \"r\", encoding=\"utf-8\") as f:\n",
    "        sents = f.read().split(\"\\n\")    \n",
    "        for i, sent in enumerate(sents):\n",
    "            line = {}\n",
    "            line[\"id\"] = str(i)\n",
    "            line[\"series\"] = title\n",
    "            line[\"text\"] = sent\n",
    "            f1.write(\"%s\\n\" % str(line))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "for t in titles:\n",
    "    passim_format(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}