text_recognition_and_geocoding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Notebook to perform text detection + recognition + geocoder queries\n",
    "\n",
    "1. Install required libraries\n",
    "2. Download pretrained models\n",
    "3. (optional) preprocess and extract map area\n",
    "4. Split into tiles\n",
    "5. Text detection + recognition\n",
    "6. Postprocess detections\n",
    "7. Geocoder querying"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import re\n",
    "import random\n",
    "import requests\n",
    "import urllib.parse\n",
    "import time\n",
    "\n",
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import cv2\n",
    "from scipy import spatial\n",
    "import tensorflow as tf\n",
    "import keras_ocr\n",
    "from shapely import geometry\n",
    "from fuzzywuzzy import fuzz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download and initialize text detection & recognition models\n",
    "\n",
    "These will be saved in the folder: pretrained_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking for pretrained_models/craft_mlt_25k.h5\n",
      "WARNING:tensorflow:From /opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/backend.py:5871: sparse_to_dense (from tensorflow.python.ops.sparse_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Create a `tf.sparse.SparseTensor` and use `tf.sparse.to_dense` instead.\n",
      "Looking for pretrained_models/crnn_kurapan.h5\n"
     ]
    }
   ],
   "source": [
    "# Initialize text detector + recognizer\n",
    "pipeline = keras_ocr.pipeline.Pipeline()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_map_area(img, verbose=0):\n",
    "    \"\"\"\n",
    "    Extract the coordinates of the black border surrounding the actual map in the image\n",
    "    :param img:\n",
    "    :param verbose:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "    gray = cv2.bitwise_not(gray)\n",
    "\n",
    "    ret, thresh = cv2.threshold(gray, 127, 255, cv2.THRESH_BINARY)\n",
    "\n",
    "    # Create structure element for extracting hor/ver lines through morphology operations\n",
    "    size = 100\n",
    "    horizontalStructure = cv2.getStructuringElement(cv2.MORPH_RECT, (size, 1))\n",
    "    verticalStructure = cv2.getStructuringElement(cv2.MORPH_RECT, (1, size))\n",
    "    # Apply morphology operations\n",
    "    horizontal = cv2.erode(thresh, horizontalStructure)\n",
    "    horizontal = cv2.dilate(horizontal, horizontalStructure)\n",
    "\n",
    "    vertical = cv2.erode(thresh, verticalStructure)\n",
    "    vertical = cv2.dilate(vertical, verticalStructure)\n",
    "    \n",
    "    merged = horizontal | vertical\n",
    "    merged = cv2.dilate(merged,np.ones((3,3)))\n",
    "    \n",
    "    rows, cols = merged.shape\n",
    "\n",
    "    top_lines = []\n",
    "    bot_lines = []\n",
    "    left_lines = []\n",
    "    right_lines = []\n",
    "\n",
    "    max_pixel_dist = 2\n",
    "    # Minimum padding for box location\n",
    "    pad_right = 100\n",
    "    pad_left = 25\n",
    "    pad_top = 100\n",
    "    pad_bot = 200\n",
    "\n",
    "    for r in range(pad_top, rows - pad_bot):\n",
    "        nonzero = cv2.countNonZero(merged[r, :])\n",
    "        # If atleast 1/8 is a line\n",
    "        if nonzero > cols // 8:\n",
    "\n",
    "            # Top\n",
    "            if r < rows // 2:\n",
    "                current_lines = top_lines\n",
    "            else:\n",
    "                current_lines = bot_lines\n",
    "\n",
    "            if len(current_lines) == 0:\n",
    "                # start_stop and count\n",
    "                current_lines.append([r, r, nonzero])\n",
    "            # If in range(forms a line)\n",
    "            # Add the right coordinate and make sum of total pixels\n",
    "            elif current_lines[-1][1] + max_pixel_dist >= r:\n",
    "                current_lines[-1][1] = r\n",
    "                current_lines[-1][2] += nonzero\n",
    "                # Found a new line outside range of previous\n",
    "            # Start a new one\n",
    "            else:\n",
    "                current_lines.append([r, r, nonzero])\n",
    "                # print('Found a line outside of pixel dist', 'r', r)\n",
    "\n",
    "    for c in range(pad_left, cols - pad_right):\n",
    "        nonzero = cv2.countNonZero(merged[:, c])\n",
    "        # If atleast half is white\n",
    "        if nonzero > rows // 8:\n",
    "\n",
    "            # Left\n",
    "            if c < cols // 2:\n",
    "                current_lines = left_lines\n",
    "            else:\n",
    "                current_lines = right_lines\n",
    "\n",
    "            if len(current_lines) == 0:\n",
    "                # start_stop and count\n",
    "                current_lines.append([c, c, nonzero])\n",
    "            # If in range(forms a line)\n",
    "            # Add the right coordinate and make sum of total pixels\n",
    "            elif current_lines[-1][1] + max_pixel_dist >= c:\n",
    "                current_lines[-1][1] = c\n",
    "                current_lines[-1][2] += nonzero\n",
    "                # Found a new line outside range of previous\n",
    "            # Start a new one\n",
    "            else:\n",
    "                current_lines.append([c, c, nonzero])\n",
    "                # print('Found a line outside of pixel dist', 'c', c)\n",
    "\n",
    "    # Take the best lines if there are multiple\n",
    "    left_lines = sorted(left_lines, key=lambda x: (x[1] - x[0], x[2]), reverse=True)\n",
    "    right_lines = sorted(right_lines, key=lambda x: (x[1] - x[0], x[2]), reverse=True)\n",
    "    top_lines = sorted(top_lines, key=lambda x: (x[1] - x[0], x[2]), reverse=True)\n",
    "    bot_lines = sorted(bot_lines, key=lambda x: (x[1] - x[0], x[2]), reverse=True)\n",
    "    \n",
    "    \n",
    "    x1, y1 = left_lines[0][0], top_lines[0][0]\n",
    "    x2, y2 = right_lines[0][1], bot_lines[0][1]\n",
    "\n",
    "    if verbose:\n",
    "        print('x1,y1', x1, y1, '\\tx2,y2', x2, y2)\n",
    "        out = img.copy()\n",
    "        cv2.rectangle(out, (x1, y1), (x2, y2), (255, 0, 0), 40)\n",
    "        plt.figure(figsize=(12,12))\n",
    "        plt.imshow(out)\n",
    "        plt.show()\n",
    "        \n",
    "    return x1,y1,x2,y2\n",
    "    \n",
    "\n",
    "def convert_predictions(predictions):\n",
    "    \"\"\"\n",
    "    Convert predictions to object style, so it works with previously developed code\n",
    "    \"\"\"\n",
    "    res = []\n",
    "    for label,bbox in predictions:\n",
    "        bbox = np.array(bbox,dtype=np.int32).flatten().tolist()\n",
    "        res.append({'text':label, 'boundingBox':bbox})\n",
    "    return res\n",
    "\n",
    "\n",
    "def map_coords(predictions,x1,y1):\n",
    "    \"\"\"\n",
    "    Translate coordinates with x1,y1\n",
    "    \"\"\"\n",
    "    if x1+y1 == 0:\n",
    "        return predictions\n",
    "    \n",
    "    res = []\n",
    "    for label,bbox in predictions:\n",
    "        bbox = np.array(bbox,dtype=np.int32)\n",
    "        bbox[:,0] += x1\n",
    "        bbox[:,1] += y1\n",
    "        res.append((label,bbox))\n",
    "        \n",
    "    return res\n",
    "\n",
    "def map_coords_word(word,x1,y1):\n",
    "    \"\"\"\n",
    "    Translate coordinates with x1,y1\n",
    "    \"\"\"\n",
    "    if x1 + y1 == 0:\n",
    "        return word\n",
    "\n",
    "    bbox = word['boundingBox']\n",
    "    bbox = [bbox[i] + x1 if i % 2 == 0 else bbox[i] + y1 for i in range(len(bbox))]\n",
    "    word['boundingBox'] = bbox\n",
    "\n",
    "    return word\n",
    "    \n",
    "\n",
    "\n",
    "def get_center(bbox):\n",
    "    \"\"\"\n",
    "    Returns the center of the given bbox\n",
    "    :param bbox:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    center_x = np.mean(bbox[::2])\n",
    "    center_y = np.mean(bbox[1::2])\n",
    "    return [center_x,center_y]\n",
    "    \n",
    "\n",
    "def merge_coords(preds):\n",
    "    \"\"\"\n",
    "    Merge the coordinates into one predictions (lines containing e or n)\n",
    "    \"\"\"\n",
    "    res = preds.copy()\n",
    "    ids_to_remove = set()\n",
    "    x_dist = 150\n",
    "    for i in range(len(preds)):\n",
    "        p = preds[i]\n",
    "        text = p['text']\n",
    "        \n",
    "        # Check for numbers to the left of this box\n",
    "        # These should always be the preceding boxes\n",
    "        if text == 'e' or text == 'n':\n",
    "            bbox = p['boundingBox']\n",
    "            center = get_center(bbox)\n",
    "            min_y = np.min(bbox[1::2])\n",
    "            max_y = np.max(bbox[1::2])\n",
    "            min_x = center[0] - x_dist\n",
    "            max_x = center[0]\n",
    "            matches = []\n",
    "            \n",
    "            for j in range(i-3,i):\n",
    "                p2 = preds[j]\n",
    "                bbox2 = p2['boundingBox']\n",
    "                center2 = get_center(bbox2)\n",
    "                if min_x < center2[0] < max_x and min_y < center2[1] < max_y:\n",
    "                    matches.append(p2)\n",
    "                    ids_to_remove.add(j)\n",
    "            \n",
    "            if len(matches) > 0:\n",
    "                matches.append(p)\n",
    "                merged = merge_matches(matches)\n",
    "                res[i] = merged\n",
    "                \n",
    "                \n",
    "    for id_ in sorted(ids_to_remove,reverse=True):\n",
    "        del res[id_]\n",
    "                    \n",
    "    return res\n",
    "\n",
    "def merge_bbox(b1, b2):\n",
    "    \"\"\"\n",
    "    Merge the two given bboxes\n",
    "    :param b1:\n",
    "    :param b2:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    # X top left, Y top left, X top right, Y top right, X bottom right, Y bottom right, X bottom left, Y bottom left\n",
    "    return [min(b1[0], b2[0]), min(b1[1], b2[1]), max(b1[2], b2[2]), min(b1[3], b2[3]),\n",
    "            max(b1[4], b2[4]), max(b1[5], b2[5]), min(b1[6], b2[6]), max(b1[7], b2[7])]\n",
    "\n",
    "def merge_matches(matches):\n",
    "    \"\"\"\n",
    "    Merge the predictions from start to end\n",
    "    \"\"\"\n",
    "    merged_text = ''\n",
    "    merged_bbox = None\n",
    "    for m in matches:\n",
    "        bbox = m['boundingBox']\n",
    "        text = m['text']\n",
    "        \n",
    "        if merged_bbox is None:\n",
    "            merged_bbox = bbox\n",
    "            merged_text = text\n",
    "        else:\n",
    "            merged_bbox = merge_bbox(merged_bbox,bbox)\n",
    "            merged_text += text\n",
    "            \n",
    "    return {'text':merged_text,'boundingBox':merged_bbox}\n",
    "\n",
    "def filter_lines(preds):\n",
    "    \"\"\"\n",
    "    Split predictions into those around the map area (greenwich and coordinates)\n",
    "    and those signifying the x/y coordinates on top. Remove the rest\n",
    "    :param lines:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    res = []\n",
    "    res_degrees = []\n",
    "    for pred in preds:\n",
    "\n",
    "        text = pred['text']\n",
    "\n",
    "        # If we find greenwich in text, we add it to degrees\n",
    "        if 'greenwich' in text:\n",
    "            pred['center'] = get_center(pred['boundingBox'])\n",
    "            res_degrees.append(pred)\n",
    "            continue\n",
    "        \n",
    "        # If we find n or e in text, we check if the rest is a number, if it is, we add it\n",
    "        # to degrees\n",
    "        if 'n' in text:\n",
    "            without = text.replace('n','')\n",
    "            try:\n",
    "                number = int(without)\n",
    "                res_degrees.append(pred)\n",
    "                continue\n",
    "            except:\n",
    "                pass\n",
    "        if 'e' in text:\n",
    "            without = text.replace('e','')\n",
    "            try:\n",
    "                number = int(without)\n",
    "                res_degrees.append(pred)\n",
    "                continue\n",
    "            except:\n",
    "                pass\n",
    "        \n",
    "        # Now we check the rest, if it is a string of length 3 and contains at least 2 numbers\n",
    "        if len(text) == 3 and sum(char.isdigit() for char in text) > 1:\n",
    "            pred['center'] = get_center(pred['boundingBox'])\n",
    "            res.append(pred)\n",
    "    \n",
    "    return res, res_degrees\n",
    "\n",
    "def rotate_predictions(preds,crop_length,crop_width,side='left'):\n",
    "    \"\"\"\n",
    "    Rotate predictions back into coordinates of original crop\n",
    "    \"\"\"\n",
    "    res = []\n",
    "    \n",
    "    for label,bbox in preds:\n",
    "        if side== 'left':\n",
    "            bbox = np.array(bbox,np.int32)\n",
    "            bbox = np.fliplr(bbox)\n",
    "            bbox[:,1] = crop_length - bbox[:,1]\n",
    "            \n",
    "        elif side == 'right':\n",
    "            bbox = np.array(bbox,np.int32)\n",
    "            bbox = np.fliplr(bbox)\n",
    "            bbox[:,0] = crop_width - bbox[:,0]\n",
    "        \n",
    "        res.append((label,bbox))\n",
    "    \n",
    "    return res\n",
    "\n",
    "\n",
    "def line_correct(lines_top, lines_bot, lines_left, lines_right):\n",
    "    \"\"\"\n",
    "    Correct the errors in the coordinates in the detected lines\n",
    "    :param lines_top:\n",
    "    :param lines_bot:\n",
    "    :param lines_left:\n",
    "    :param lines_right:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    # sort lines in asc order\n",
    "    top = sorted(lines_top, key=lambda x: x['center'][0])\n",
    "    bot = sorted(lines_bot, key=lambda x: x['center'][0])\n",
    "    left = sorted(lines_left, key=lambda x: x['center'][1], reverse=True)\n",
    "    right = sorted(lines_right, key=lambda x: x['center'][1], reverse=True)\n",
    "    \n",
    "    # Replace o by 0\n",
    "    for l in (top,bot,left,right):\n",
    "        for l2 in l:\n",
    "            l2['text'] = l2['text'].replace('o','0')\n",
    "    \n",
    "    print('top lines:',[t['text'] for t in top])\n",
    "    print('bot lines:',[t['text'] for t in bot])\n",
    "    print('left lines:',[t['text'] for t in left])\n",
    "    print('right lines:',[t['text'] for t in right])\n",
    "    # Correct top & bot\n",
    "    avg_factor_top, avg_p_t, avg_c_t = get_coord_dist(top, 0)\n",
    "    avg_factor_bot, avg_p_b, avg_c_b = get_coord_dist(bot, 0)\n",
    "\n",
    "    x_factor = np.mean((avg_factor_top, avg_factor_bot))\n",
    "    avg_p_x = np.mean((avg_p_t, avg_p_b))\n",
    "    avg_c_x = np.mean((avg_c_t, avg_c_b))\n",
    "\n",
    "    # Correct left & right\n",
    "    avg_factor_left, avg_p_l, avg_c_l = get_coord_dist(left, 1)\n",
    "    avg_factor_right, avg_p_r, avg_c_r = get_coord_dist(right, 1)\n",
    "    y_factor = np.mean((avg_factor_left, avg_factor_right))\n",
    "\n",
    "    avg_p_y = np.mean((avg_p_l, avg_p_r))\n",
    "    avg_c_y = np.mean((avg_c_l, avg_c_r))\n",
    "\n",
    "    conversion_factor = np.array((-y_factor, x_factor))\n",
    "    avg_point = np.array((avg_p_y, avg_p_x))\n",
    "    avg_coords = np.array((avg_c_y, avg_c_x))\n",
    "    zero_point_coords = avg_coords - (avg_point * conversion_factor)\n",
    "\n",
    "    return zero_point_coords, conversion_factor\n",
    "\n",
    "\n",
    "def get_coord_dist(lines, center_index):\n",
    "    \"\"\"\n",
    "    Get the average coordinate distance over pixel distance factor, average pixel coordinate, and average coordinate\n",
    "    :param lines:\n",
    "    :param center_index:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    coords = []\n",
    "    pixel_coords = []\n",
    "    for l in lines:\n",
    "        c = l['text']\n",
    "        try:\n",
    "            c = int(c)\n",
    "            coords.append(c)\n",
    "            pixel_coords.append(l['center'][center_index])\n",
    "        except ValueError:\n",
    "            pass\n",
    "\n",
    "    median_c = np.median(coords)\n",
    "    ids_to_remove = []\n",
    "    # remove outliers\n",
    "    for i, c in enumerate(coords):\n",
    "        if abs(c - median_c) > 10:\n",
    "            ids_to_remove.append(i)\n",
    "    for i in sorted(ids_to_remove, reverse=True):\n",
    "        del coords[i]\n",
    "        del pixel_coords[i]\n",
    "\n",
    "    d_coords = np.abs(np.diff(coords))\n",
    "    d_pixel_coords = np.abs(np.diff(pixel_coords))\n",
    "\n",
    "    return np.mean(d_coords / d_pixel_coords), np.mean(pixel_coords), np.mean(coords)\n",
    "\n",
    "\n",
    "def extract_coordinate_area(img,map_info,x1,y1,x2,y2):\n",
    "    \"\"\"\n",
    "    Extract the actual map area and the pixel to coordinate transform\n",
    "    \"\"\"\n",
    "    crop_width = 150\n",
    "    crop_length = 2000\n",
    "    # crop of whole top\n",
    "\n",
    "    all_preds = []\n",
    "\n",
    "    # Top and bottom first\n",
    "    current_x = x1\n",
    "    while current_x < x2:\n",
    "        print('Current x:',current_x)\n",
    "        x_stop = min(x2, current_x + crop_length)\n",
    "\n",
    "        crop_top = img[y1:y1 + crop_width, current_x:x_stop]\n",
    "        preds = pipeline.recognize([crop_top])[0]\n",
    "        preds = map_coords(preds, current_x, y1)\n",
    "        preds = convert_predictions(preds)\n",
    "        all_preds.extend(preds)\n",
    "\n",
    "        crop_bot = img[y2 - crop_width:y2, current_x:x_stop]\n",
    "        preds = pipeline.recognize([crop_bot])[0]\n",
    "        preds = map_coords(preds, current_x, y2 - crop_width)\n",
    "        preds = convert_predictions(preds)\n",
    "        all_preds.extend(preds)\n",
    "        current_x += crop_length\n",
    "\n",
    "    # Filter out lines with numbers\n",
    "    all_lines = merge_coords(all_preds)\n",
    "    filtered_lines, degrees_lines = filter_lines(all_lines)\n",
    "    \n",
    "    \n",
    "    top_lines = [l for l in filtered_lines if l['center'][1] < (y2 - y1) // 2]\n",
    "    bot_lines = [l for l in filtered_lines if l['center'][1] > (y2 - y1) // 2]\n",
    "\n",
    "    delta_y = 10\n",
    "\n",
    "    possible_lines_top = []\n",
    "    possible_ids_top = []\n",
    "    possible_lines_bot = []\n",
    "    possible_ids_bot = []\n",
    "\n",
    "    top_y = [l['center'][1] for l in top_lines]\n",
    "    bot_y = [l['center'][1] for l in bot_lines]\n",
    "\n",
    "    # TODO Fix code duplication\n",
    "\n",
    "    for i, y in enumerate(top_y):\n",
    "        if len(possible_lines_top) == 0:\n",
    "            possible_lines_top = [[y]]\n",
    "            possible_ids_top = [[i]]\n",
    "        else:\n",
    "\n",
    "            average_y = [abs(y - np.mean(item)) for item in possible_lines_top]\n",
    "            if min(average_y) < delta_y:\n",
    "                idx = np.argmin(average_y)\n",
    "                possible_lines_top[idx].append(y)\n",
    "                possible_ids_top[idx].append(i)\n",
    "            else:\n",
    "                possible_lines_top.append([y])\n",
    "                possible_ids_top.append([i])\n",
    "\n",
    "    for i, y in enumerate(bot_y):\n",
    "        if len(possible_lines_bot) == 0:\n",
    "            possible_lines_bot = [[y]]\n",
    "            possible_ids_bot = [[i]]\n",
    "        else:\n",
    "\n",
    "            average_y = [abs(y - np.mean(item)) for item in possible_lines_bot]\n",
    "            if min(average_y) < delta_y:\n",
    "                idx = np.argmin(average_y)\n",
    "                possible_lines_bot[idx].append(y)\n",
    "                possible_ids_bot[idx].append(i)\n",
    "            else:\n",
    "                possible_lines_bot.append([y])\n",
    "                possible_ids_bot.append([i])\n",
    "\n",
    "    ids_top = possible_ids_top[np.argmax([len(i) for i in possible_ids_top])]\n",
    "    lines_top = [top_lines[i] for i in ids_top]\n",
    "    ids_bot = possible_ids_bot[np.argmax([len(i) for i in possible_ids_bot])]\n",
    "    lines_bot = [bot_lines[i] for i in ids_bot]\n",
    "\n",
    "    # Do the same for left/right\n",
    "\n",
    "    all_lines2 = []\n",
    "    current_y = y1\n",
    "\n",
    "    while current_y < y2:\n",
    "        print('Current y', current_y)\n",
    "        y_stop = min(y2, current_y + crop_length)\n",
    "\n",
    "        crop_left = img[current_y:y_stop, x1:x1 + crop_width]\n",
    "        # Rotate image so text is upright\n",
    "        rot = np.rot90(crop_left,axes=(1,0))\n",
    "        preds = pipeline.recognize([rot])[0]\n",
    "        # Rotate predictions back to original image\n",
    "\n",
    "        preds = rotate_predictions(preds,crop_length,crop_width,side='left')\n",
    "        preds = map_coords(preds, x1, current_y)\n",
    "        preds = convert_predictions(preds)\n",
    "        all_lines2.extend(preds)\n",
    "\n",
    "        crop_right = img[current_y:y_stop, x2 - crop_width:x2]\n",
    "        # Rotate image so text is upright\n",
    "        rot = np.rot90(crop_right)\n",
    "        preds = pipeline.recognize([rot])[0]\n",
    "        # Rotate predictions back to original image\n",
    "        preds = rotate_predictions(preds,crop_length,crop_width,side='right')\n",
    "        preds = map_coords(preds, x2 - crop_width,current_y)\n",
    "        preds = convert_predictions(preds)\n",
    "        all_lines2.extend(preds)\n",
    "\n",
    "        current_y += crop_length\n",
    "    \n",
    "    \n",
    "    all_lines2 = merge_coords(all_lines2)\n",
    "    '''\n",
    "    print('All lines (left/right):')\n",
    "    for l in all_lines2:\n",
    "        print(l['text'],get_center(l['boundingBox']))\n",
    "    '''\n",
    "    filtered_lines, degrees_lines2 = filter_lines(all_lines2)\n",
    "    \n",
    "    left_lines = [l for l in filtered_lines if l['center'][0] < (x2 - x1) // 2]\n",
    "    right_lines = [l for l in filtered_lines if l['center'][0] > (x2 - x1) // 2]\n",
    "    '''\n",
    "    print('Filtered lines (left/right):')\n",
    "    for l in filtered_lines:\n",
    "        print(l['text'],l['center'])\n",
    "    '''\n",
    "    \n",
    "    delta_x = 10\n",
    "\n",
    "    possible_lines_left = []\n",
    "    possible_ids_left = []\n",
    "    possible_lines_right = []\n",
    "    possible_ids_right = []\n",
    "\n",
    "    left_x = [l['center'][0] for l in left_lines]\n",
    "    right_x = [l['center'][0] for l in right_lines]\n",
    "\n",
    "    for i, x in enumerate(left_x):\n",
    "        if len(possible_lines_left) == 0:\n",
    "            possible_lines_left = [[x]]\n",
    "            possible_ids_left = [[i]]\n",
    "        else:\n",
    "\n",
    "            average_x = [abs(x - np.mean(item)) for item in possible_lines_left]\n",
    "            if min(average_x) < delta_x:\n",
    "                idx = np.argmin(average_x)\n",
    "                possible_lines_left[idx].append(x)\n",
    "                possible_ids_left[idx].append(i)\n",
    "            else:\n",
    "                possible_lines_left.append([x])\n",
    "                possible_ids_left.append([i])\n",
    "\n",
    "    for i, x in enumerate(right_x):\n",
    "        if len(possible_lines_right) == 0:\n",
    "            possible_lines_right = [[x]]\n",
    "            possible_ids_right = [[i]]\n",
    "        else:\n",
    "\n",
    "            average_x = [abs(x - np.mean(item)) for item in possible_lines_right]\n",
    "            if min(average_x) < delta_x:\n",
    "                idx = np.argmin(average_x)\n",
    "                possible_lines_right[idx].append(x)\n",
    "                possible_ids_right[idx].append(i)\n",
    "            else:\n",
    "                possible_lines_right.append([x])\n",
    "                possible_ids_right.append([i])\n",
    "\n",
    "    ids_left = possible_ids_left[np.argmax([len(i) for i in possible_ids_left])]\n",
    "    lines_left = [left_lines[i] for i in ids_left]\n",
    "    ids_right = possible_ids_right[np.argmax([len(i) for i in possible_ids_right])]\n",
    "    lines_right = [right_lines[i] for i in ids_right]\n",
    "\n",
    "    # Determine pixel to Lambert72 conversion factors\n",
    "    c_zero, factor = line_correct(lines_top, lines_bot, lines_left, lines_right)\n",
    "    print('Coordinates for 0,0:', c_zero)\n",
    "    map_info['c_zero'] = c_zero.tolist()\n",
    "    map_info['factor'] = factor.tolist()\n",
    "\n",
    "\n",
    "    # Now find the 'actual' map area by looking at the lines with degrees as they are closest to the actual map area\n",
    "\n",
    "    for l in degrees_lines:\n",
    "        l['center'] = get_center(l['boundingBox'])\n",
    "\n",
    "    d_top = [l for l in degrees_lines if l['center'][1] < (y2 - y1) // 2]\n",
    "    d_bot = [l for l in degrees_lines if l['center'][1] > (y2 - y1) // 2]\n",
    "    \n",
    "    if len(d_top) == 0:\n",
    "        d_top = [l for l in get_backup_degrees_lines(all_lines,direction='up') if l['center'][1] < (y2 - y1) // 2]\n",
    "    \n",
    "    if len(d_bot) == 0:\n",
    "        d_bot = [l for l in get_backup_degrees_lines(all_lines,direction='up') if l['center'][1] > (y2 - y1) // 2]\n",
    "    \n",
    "    # Here we try to detect greenwich or the degrees with N and E\n",
    "    # to determine the effective map area\n",
    "\n",
    "    # Find one correct degree\n",
    "    found = False\n",
    "    y_top = 0\n",
    "    # Check for greenwich\n",
    "    for l in d_top:\n",
    "        if 'greenwich' in l['text'].lower():\n",
    "            found = True\n",
    "            y_top = np.max(l['boundingBox'][1::2])\n",
    "            break\n",
    "\n",
    "    # Check for N in combination with 6 digits for lines top\n",
    "    if not found:\n",
    "        for l in d_top:\n",
    "            text = l['text'].lower()\n",
    "\n",
    "            if text.endswith('n') and len(text) in {6,7}:\n",
    "                # check if numbers are before it\n",
    "                try:\n",
    "                    n = int(text[:-1])\n",
    "                    found = True\n",
    "                    y_top = np.max(l['boundingBox'][1::2])\n",
    "                    break\n",
    "                except ValueError:\n",
    "                    pass\n",
    "\n",
    "    # Else take the average of min of the boxes\n",
    "    if not found:\n",
    "        for l in d_top:\n",
    "            y_top += np.max(l['boundingBox'][1::2])\n",
    "        y_top /= len(d_top)\n",
    "\n",
    "    # Find one correct degree\n",
    "    found = False\n",
    "    y_bot = 0\n",
    "    # Check for greenwich\n",
    "    for l in d_bot:\n",
    "        if 'greenwich' in l['text'].lower():\n",
    "            found = True\n",
    "            y_bot = np.min(l['boundingBox'][1::2])\n",
    "            break\n",
    "    # Check for ' or \" in combination with -N for lines top\n",
    "    if not found:\n",
    "        for l in d_bot:\n",
    "            text = l['text'].lower()\n",
    "            if text.endswith('n') and len(text) in {6,7}:\n",
    "                # check if numbers are before it\n",
    "                try:\n",
    "                    n = int(text[:-1])\n",
    "                    found = True\n",
    "                    y_bot = np.min(l['boundingBox'][1::2])\n",
    "                    break\n",
    "                except ValueError:\n",
    "                    pass\n",
    "    # Else take the average of min of the boxes\n",
    "    if not found:\n",
    "        for l in d_bot:\n",
    "            y_bot += np.min(l['boundingBox'][1::2])\n",
    "        y_bot /= len(d_bot)\n",
    "\n",
    "    # Now left/right\n",
    "\n",
    "    for l in degrees_lines2:\n",
    "        l['center'] = get_center(l['boundingBox'])\n",
    "    \n",
    "    \n",
    "    d_left = [l for l in degrees_lines2 if l['center'][0] < (x2 - x1) // 2]\n",
    "    d_right = [l for l in degrees_lines2 if l['center'][0] > (x2 - x1) // 2]\n",
    "    \n",
    "    if len(d_left) == 0:\n",
    "        d_left = [l for l in get_backup_degrees_lines(all_lines2,direction='left') if l['center'][0] < (x2 - x1) // 2]\n",
    "    \n",
    "    if len(d_right) == 0:\n",
    "        d_right = [l for l in get_backup_degrees_lines(all_lines2,direction='left') if l['center'][0] > (x2 - x1) // 2]\n",
    "    \n",
    "    # Find one correct degree\n",
    "    found = False\n",
    "    x_left = 0\n",
    "\n",
    "    # Check for ' or \" in combination with -N or east for lines left\n",
    "    if not found:\n",
    "        for l in d_left:\n",
    "            text = l['text'].lower()\n",
    "\n",
    "            if text.endswith('e') and len(text) in {6,7}:\n",
    "                # check if numbers are before it\n",
    "                try:\n",
    "                    n = int(text[:-1])\n",
    "                    found = True\n",
    "                    x_left = np.max(l['boundingBox'][::2])\n",
    "                    break\n",
    "                except ValueError:\n",
    "                    pass\n",
    "\n",
    "    # Else take the average of min of the boxes\n",
    "    if not found:\n",
    "        for l in d_left:\n",
    "            x_left += np.max(l['boundingBox'][::2])\n",
    "        x_left /= len(d_left)\n",
    "\n",
    "    # Find one correct degree\n",
    "    found = False\n",
    "    x_right = 0\n",
    "\n",
    "    # Check for ' or \" in combination with -N for lines top\n",
    "    if not found:\n",
    "        for l in d_right:\n",
    "            text = l['text'].lower()\n",
    "\n",
    "            if text.endswith('e') and len(text) in {6,7}:\n",
    "                # check if numbers are before it\n",
    "                try:\n",
    "                    n = int(text[:-1])\n",
    "                    found = True\n",
    "                    x_right = np.min(l['boundingBox'][::2])\n",
    "                    break\n",
    "                except ValueError:\n",
    "                    pass\n",
    "\n",
    "    # Else take the average of min of the boxes\n",
    "    if not found:\n",
    "        for l in d_right:\n",
    "            x_right += np.min(l['boundingBox'][::2])\n",
    "        x_right /= len(d_right)\n",
    "        \n",
    "    map_info['map_area'] = {'x1': int(x_left), 'y1': int(y_top), 'x2': int(x_right), 'y2': int(y_bot)}\n",
    "    map_info['coord_lines_top'] = lines_top\n",
    "    map_info['coord_lines_bot'] = lines_bot\n",
    "    map_info['coord_lines_left'] = lines_left\n",
    "    map_info['coord_lines_right'] = lines_right\n",
    "    \n",
    "    return map_info\n",
    "\n",
    "\n",
    "def write_predictions(out_paths,prediction_groups):\n",
    "    # if only 1 path and prediction is given\n",
    "    if isinstance(out_paths,str):\n",
    "        out_paths = [out_paths]\n",
    "        prediction_groups = [prediction_groups]\n",
    "            \n",
    "    for out_path,pred_group in zip(out_paths,prediction_groups):\n",
    "        words = []\n",
    "        \n",
    "        for text,bbox in pred_group:\n",
    "            bbox = np.int32(np.round(bbox)).ravel().tolist()\n",
    "            obj = {'text':text,'boundingBox':bbox}\n",
    "            words.append(obj)\n",
    "        \n",
    "        json.dump({'words':words},open(out_path,'w',encoding='utf-8'))\n",
    "        \n",
    "def iou(a, b):\n",
    "    a = geometry.Polygon([(a[0], a[1]), (a[2], a[3]), (a[4], a[5]), (a[6], a[7])])\n",
    "    b = geometry.Polygon([(b[0], b[1]), (b[2], b[3]), (b[4], b[5]), (b[6], b[7])])\n",
    "\n",
    "    return a.intersection(b).area / a.union(b).area\n",
    "\n",
    "\n",
    "def get_backup_degrees_lines(lines,direction='up'):\n",
    "    \"\"\"\n",
    "    Loosen restrictions to get degrees lines\n",
    "    Just look for number combinations of length 4 or higher and look for n or e\n",
    "    \"\"\"\n",
    "    degrees_lines = []\n",
    "    char = 'n' if direction=='up' else 'e'\n",
    "    for l in lines:\n",
    "        text = l['text'].lower()\n",
    "        center = get_center(l['boundingBox'])\n",
    "        l['center'] = center\n",
    "        \n",
    "        if direction == 'up' and text == char:\n",
    "            degrees_lines.append(l)\n",
    "        \n",
    "        elif len(text) > 3:\n",
    "            try:\n",
    "                num = int(text)\n",
    "                degrees_lines.append(l)\n",
    "            except:\n",
    "                pass\n",
    "    return degrees_lines\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Detect effective map region (not needed for TOP50Raster)\n",
    "\n",
    "### Split into tiles (2500x2500 with 500 overlap)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = 'dataset/TOP50raster'\n",
    "# if img_dir == crop_dir, no preprocessing will occurr\n",
    "img_dir = os.path.join(data_dir,'cropped')\n",
    "crop_dir = os.path.join(data_dir,'cropped')\n",
    "json_dir = os.path.join(data_dir,'json')\n",
    "tile_dir = os.path.join(data_dir,'tiles')\n",
    "\n",
    "tile_size = 2000\n",
    "overlap = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current map: TOP50raster-31O-2018\n",
      "\n",
      "Current map: TOP50raster-32O-2018\n",
      "\n",
      "Current map: TOP50raster-32W-2018\n",
      "\n",
      "Current map: TOP50raster-38O-2018\n",
      "\n",
      "Current map: TOP50raster-39O-2018\n",
      "\n",
      "Current map: TOP50raster-39W-2018\n",
      "\n",
      "Current map: TOP50raster-44O-2018\n",
      "\n",
      "Current map: TOP50raster-45O-2018\n",
      "\n",
      "Current map: TOP50raster-45W-2018\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for fname in os.listdir(img_dir):\n",
    "    if fname.endswith('.jpg'):\n",
    "        \n",
    "        map_name = fname.replace('.jpg','')\n",
    "        print('Current map:',map_name)\n",
    "        lines_fname = os.path.join(json_dir,fname.replace('.jpg','_lines.json'))\n",
    "        \n",
    "        # Check if the map has been fully processed before\n",
    "        if os.path.exists(lines_fname):\n",
    "            print('Map already processed')\n",
    "            continue\n",
    "        \n",
    "        json_fname = os.path.join(json_dir,fname.replace('.jpg','.json'))\n",
    "        try:\n",
    "            map_info = json.load(open(json_fname,'r'))\n",
    "        except FileNotFoundError:\n",
    "            map_info = {}\n",
    "        \n",
    "        crop_path = os.path.join(img_dir,fname)\n",
    "        orig_img = cv2.imread(os.path.join(img_dir,fname))\n",
    "        \n",
    "        # If map still needs to be cropped (not the case for TOP50raster)\n",
    "        if not os.path.exists(crop_path):\n",
    "            img = cv2.cvtColor(orig_img,cv2.COLOR_BGR2RGB)\n",
    "            x1,y1,x2,y2 = extract_map_area(img,verbose=0)\n",
    "            map_info = extract_coordinate_area(img,map_info,x1,y1,x2,y2)\n",
    "            \n",
    "            map_area = map_info['map_area']\n",
    "            x1 = map_area['x1']\n",
    "            y1 = map_area['y1']\n",
    "            x2 = map_area['x2']\n",
    "            y2 = map_area['y2']\n",
    "            crop = orig_img[y1:y2,x1:x2]\n",
    "            cv2.imwrite(crop_path,crop)\n",
    "            json.dump(map_info, open(json_fname, 'w'))\n",
    "        \n",
    "        # Split into tiles and save them in tiles dir\n",
    "        img = orig_img\n",
    "        x1,x2,y1,y2 = 0,0,0,0\n",
    "        h, w, _ = img.shape\n",
    "        if y2 == 0:\n",
    "            y2 = h\n",
    "        if x2 == 0:\n",
    "            x2 = w\n",
    "        row_index = 0\n",
    "        for r in range(y1, y2, tile_size):\n",
    "            col_index = 0\n",
    "            for c in range(x1, x2, tile_size):\n",
    "                if r > h or c > w:\n",
    "                    # This should never execute\n",
    "                    print('Something went wrong')\n",
    "                    continue\n",
    "\n",
    "                r_max = min(y2, r + tile_size + overlap)\n",
    "                c_max = min(x2, c + tile_size + overlap)\n",
    "                tile = img[r:r_max, c:c_max]\n",
    "                fname_tile = os.path.join(tile_dir, map_name + '_tile_' + str(row_index) + '_' + str(col_index) + '.jpg')\n",
    "                cv2.imwrite(fname_tile, tile)\n",
    "                col_index += 1\n",
    "            row_index += 1\n",
    "        print()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Detect and recognize text in each tile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current: TOP50raster-31O-2018\n",
      "Recognition preprocess: 1.733 s\n",
      "Recognition predict: 20.214 s\n",
      "\n",
      "Current: TOP50raster-32O-2018\n",
      "Recognition preprocess: 1.697 s\n",
      "Recognition predict: 12.424 s\n",
      "\n",
      "Current: TOP50raster-32W-2018\n",
      "Recognition preprocess: 1.594 s\n",
      "Recognition predict: 11.085 s\n",
      "\n",
      "Current: TOP50raster-38O-2018\n",
      "Recognition preprocess: 2.036 s\n",
      "Recognition predict: 12.618 s\n",
      "\n",
      "Current: TOP50raster-39O-2018\n",
      "Recognition preprocess: 2.131 s\n",
      "Recognition predict: 13.054 s\n",
      "\n",
      "Current: TOP50raster-39W-2018\n",
      "Recognition preprocess: 2.487 s\n",
      "Recognition predict: 2.809 s\n",
      "\n",
      "Current: TOP50raster-44O-2018\n",
      "Recognition preprocess: 2.028 s\n",
      "Recognition predict: 13.049 s\n",
      "\n",
      "Current: TOP50raster-45O-2018\n",
      "Recognition preprocess: 1.91 s\n",
      "Recognition predict: 2.13 s\n",
      "\n",
      "Current: TOP50raster-45W-2018\n",
      "Recognition preprocess: 2.267 s\n",
      "Recognition predict: 14.396 s\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for fname in os.listdir(img_dir):\n",
    "    if fname.endswith('.jpg'):\n",
    "        map_name = fname.replace('.jpg','')\n",
    "        print('Current:',map_name)\n",
    "        lines_fname = os.path.join(json_dir,fname.replace('.jpg','_lines.json'))\n",
    "        \n",
    "        # Check if the map has been fully processed before\n",
    "        if os.path.exists(lines_fname):\n",
    "            print('Map already processed')\n",
    "            continue\n",
    "        tile_fnames = [os.path.join(tile_dir,f) for f in os.listdir(tile_dir) if f.endswith('.jpg') and map_name in f]\n",
    "        json_fnames = [f.replace('.jpg','_words.json') for f in tile_fnames]\n",
    "        \n",
    "        images = [keras_ocr.tools.read(f) for f in tile_fnames]\n",
    "        \n",
    "        pred_groups = pipeline.recognize(images)\n",
    "        write_predictions(json_fnames,pred_groups)\n",
    "        print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize predictions on first tile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fname = 'TOP50raster-31O-2018_tile_0_0.jpg'\n",
    "img = cv2.imread(os.path.join(tile_dir,fname))\n",
    "words = json.load(open(os.path.join(tile_dir,fname.replace('.jpg','_words.json')), 'r'))['words']\n",
    "\n",
    "for w in words:\n",
    "    text = w['text']\n",
    "    bbox = np.array(w['boundingBox'],dtype=np.int32)\n",
    "    bbox = bbox.reshape((4,2))\n",
    "    \n",
    "    x1, y1 = np.min(bbox,axis=0)-2\n",
    "    cv2.polylines(img, np.int32([bbox]), True, (0, 0, 255), 3)\n",
    "    cv2.putText(img, text, (x1, y1), cv2.FONT_HERSHEY_PLAIN, 2, (225, 0, 0), 2, cv2.LINE_AA)\n",
    "cv2.imwrite('TOP50raster-31O-2018_tile_0_0_detections.jpg',img)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Merge predictions from the tiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current: TOP50raster-31O-2018\n",
      "\n",
      "Average words in each tile: 61.05\n",
      "Current: TOP50raster-32O-2018\n",
      "\n",
      "Average words in each tile: 59.75\n",
      "Current: TOP50raster-32W-2018\n",
      "\n",
      "Average words in each tile: 54.25\n",
      "Current: TOP50raster-38O-2018\n",
      "\n",
      "Average words in each tile: 70.45\n",
      "Current: TOP50raster-39O-2018\n",
      "\n",
      "Average words in each tile: 73.55\n",
      "Current: TOP50raster-39W-2018\n",
      "\n",
      "Average words in each tile: 86.6\n",
      "Current: TOP50raster-44O-2018\n",
      "\n",
      "Average words in each tile: 72.1\n",
      "Current: TOP50raster-45O-2018\n",
      "\n",
      "Average words in each tile: 66.3\n",
      "Current: TOP50raster-45W-2018\n",
      "\n",
      "Average words in each tile: 79.85\n"
     ]
    }
   ],
   "source": [
    "for fname in os.listdir(img_dir):\n",
    "    if fname.endswith('.jpg'):\n",
    "        map_name = fname.replace('.jpg','')\n",
    "        print('Current:',map_name)\n",
    "        lines_fname = os.path.join(json_dir,fname.replace('.jpg','_lines.json'))\n",
    "        \n",
    "        # Check if the map has been fully processed before\n",
    "        if os.path.exists(lines_fname):\n",
    "            print('Map already processed')\n",
    "            continue\n",
    "        tile_fnames = [os.path.join(tile_dir,f) for f in os.listdir(tile_dir) if map_name in f]\n",
    "        \n",
    "        max_r = 0\n",
    "        max_c = 0\n",
    "\n",
    "        for tile_name in tile_fnames:\n",
    "            r = tile_name.split('_tile_')[1][0]\n",
    "            c = tile_name.split('_tile_' + str(r) + '_')[1][0]\n",
    "\n",
    "            if int(r) > max_r:\n",
    "                max_r = int(r)\n",
    "            if int(c) > max_c:\n",
    "                max_c = int(c)\n",
    "        \n",
    "        # Make 2D matrix to compare adjacent tiles\n",
    "        words_matrix = [[None] * (max_c + 1) for i in range(max_r + 1)]\n",
    "\n",
    "        for r in range(max_r + 1):\n",
    "            for c in range(max_c + 1):\n",
    "                delta_y = tile_size * r\n",
    "                delta_x = tile_size * c\n",
    "                \n",
    "                pred_fname = os.path.join(tile_dir,map_name + '_tile_' + str(r) + '_' + str(c) + '_words.json')\n",
    "                words = json.load(open(pred_fname,'r'))['words']\n",
    "                tile_words = []\n",
    "                for w in words:\n",
    "                    bbox = w['boundingBox']\n",
    "                    # Filter on words inside region without overlap\n",
    "                    if bbox[0] <= tile_size and bbox[1] <= tile_size:\n",
    "                        w = map_coords_word(w, delta_x, delta_y)\n",
    "                        tile_words.append(w)\n",
    "\n",
    "                words_matrix[r][c] = tile_words\n",
    "\n",
    "        # Remove symbols to merge text\n",
    "        pat = re.compile(r'[-.,;:?!_()\\[\\]{\\}&°+%€/]+')\n",
    "        max_y = 0\n",
    "        total_words = 0\n",
    "        for r in range(max_r + 1):\n",
    "            max_y += tile_size\n",
    "            max_x = 0\n",
    "            for c in range(max_c + 1):\n",
    "                max_x += tile_size\n",
    "                current_words = words_matrix[r][c]\n",
    "                total_words += len(current_words)\n",
    "                current_candidates_right = []\n",
    "                current_candidates_below = []\n",
    "\n",
    "                # Filter candidates first to avoid unneccessary loops\n",
    "                for i, word in enumerate(current_words):\n",
    "                    bbox = word['boundingBox']\n",
    "\n",
    "                    if c < max_c and (bbox[2] > max_x or bbox[4] > max_x):\n",
    "                        current_candidates_right.append((i, word))\n",
    "                    if r + 1 < max_r and (bbox[5] > max_y or bbox[7] > max_y):\n",
    "                        current_candidates_below.append((i, word))\n",
    "\n",
    "                # Check the right\n",
    "                if c < max_c:\n",
    "                    next_words = words_matrix[r][c + 1]\n",
    "                    words_to_remove = []\n",
    "                    candidates = []\n",
    "                    # Filter candidates first to avoid unneccessary loops\n",
    "                    for i, word in enumerate(next_words):\n",
    "                        bbox = word['boundingBox']\n",
    "\n",
    "                        if bbox[2] < max_x + overlap or bbox[4] < max_x + overlap:\n",
    "                            candidates.append((i, word))\n",
    "                    # TODO: fix code duplication\n",
    "                    # For each line in current candidates, see if it intersects with a candidate from next tile\n",
    "                    for current_candidate in current_candidates_right:\n",
    "                        current_index = current_candidate[0]\n",
    "                        current_bbox = current_candidate[1]['boundingBox']\n",
    "\n",
    "                        for candidate in candidates:\n",
    "                            index = candidate[0]\n",
    "\n",
    "                            bbox = candidate[1]['boundingBox']\n",
    "\n",
    "                            if iou(current_bbox, bbox) > 0:\n",
    "                                current_text = re.sub(pat, '', current_candidate[1]['text'].lower())\n",
    "                                text = re.sub(pat, '', candidate[1]['text'].lower())\n",
    "                                partial_ratio = fuzz.partial_ratio(current_text, text)\n",
    "\n",
    "                                # match, replace text in current by longest of the two,delete line in other\n",
    "                                if partial_ratio > 60:\n",
    "                                    # print('match', text, '+', current_text)\n",
    "                                    words_to_remove.append(index)\n",
    "                                    # replace current by other\n",
    "                                    if len(current_text) < len(text):\n",
    "                                        current_words[current_index] = next_words[index]\n",
    "\n",
    "                                    break\n",
    "\n",
    "                    words_matrix[r][c] = current_words\n",
    "                    for i in sorted(words_to_remove, reverse=True):\n",
    "                        # print('Deleting', lines_matrix[r][c + 1][i]['text'])\n",
    "                        del words_matrix[r][c + 1][i]\n",
    "\n",
    "                # Check the bottom\n",
    "                if r < max_r:\n",
    "                    next_words = words_matrix[r + 1][c]\n",
    "                    words_to_remove = []\n",
    "                    candidates = []\n",
    "                    # Filter candidates first to avoid unneccessary loops\n",
    "                    for i, word in enumerate(next_words):\n",
    "                        bbox = word['boundingBox']\n",
    "\n",
    "                        if bbox[5] < max_y + overlap or bbox[7] < max_y + overlap:\n",
    "                            candidates.append((i, word))\n",
    "\n",
    "                    # For each line in current candidates, see if it intersects with a candidate from next tile\n",
    "                    for current_candidate in current_candidates_below:\n",
    "                        current_index = current_candidate[0]\n",
    "                        current_bbox = current_candidate[1]['boundingBox']\n",
    "\n",
    "                        for candidate in candidates:\n",
    "                            index = candidate[0]\n",
    "                            bbox = candidate[1]['boundingBox']\n",
    "\n",
    "                            if iou(current_bbox, bbox) > 0:\n",
    "                                current_text = re.sub(pat, '', current_candidate[1]['text'].lower())\n",
    "                                text = re.sub(pat, '', candidate[1]['text'].lower())\n",
    "                                partial_ratio = fuzz.partial_ratio(current_text, text)\n",
    "\n",
    "                                # match, replace text in current by longest of the two,delete line in other\n",
    "                                if partial_ratio > 60:\n",
    "                                    # print('match', text, '+', current_text)\n",
    "                                    words_to_remove.append(index)\n",
    "                                    # replace current by other\n",
    "                                    if len(current_text) < len(text):\n",
    "                                        current_words[current_index] = next_words[index]\n",
    "\n",
    "                                    break\n",
    "\n",
    "                    words_matrix[r][c] = current_words\n",
    "\n",
    "                    for i in sorted(words_to_remove, reverse=True):\n",
    "                        # print('Deleting', lines_matrix[r + 1][c][i]['text'])\n",
    "                        del words_matrix[r + 1][c][i]\n",
    "\n",
    "        print('\\nAverage words in each tile:', total_words / ((max_c + 1) * (max_r + 1)))\n",
    "        words = []\n",
    "        \n",
    "        for r in range(max_r + 1):\n",
    "            for c in range(max_c + 1):\n",
    "                words.extend(words_matrix[r][c])\n",
    "               \n",
    "                tile_path = os.path.join(tile_dir,map_name+'_tile_' + str(r) + '_' + str(c) + '.jpg')\n",
    "                tile_w_path = os.path.join(tile_dir,map_name+'_tile_' + str(r) + '_' + str(c) + '_words.json')\n",
    "                try:\n",
    "                    os.remove(tile_path)\n",
    "                    os.remove(tile_w_path)\n",
    "                except FileNotFoundError:\n",
    "                    pass\n",
    "                \n",
    "                \n",
    "        words_path = os.path.join(json_dir,map_name+'_words.json')\n",
    "        json.dump({'words': words}, open(words_path, 'w'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show(img):\n",
    "    return plt.imshow(cv2.cvtColor(img,cv2.COLOR_BGR2RGB))\n",
    "\n",
    "def get_rotated_width_height(box):\n",
    "    \"\"\"\n",
    "    Returns the width and height of a rotated bounding box\n",
    "\n",
    "    Args:\n",
    "        box: A list of four points starting in the top left\n",
    "        corner and moving clockwise.\n",
    "    \"\"\"\n",
    "    w = (spatial.distance.cdist(box[0][np.newaxis], box[1][np.newaxis], \"euclidean\") + spatial.distance.cdist(\n",
    "        box[2][np.newaxis], box[3][np.newaxis], \"euclidean\")) / 2\n",
    "    h = (spatial.distance.cdist(box[0][np.newaxis], box[3][np.newaxis], \"euclidean\") + spatial.distance.cdist(\n",
    "        box[1][np.newaxis], box[2][np.newaxis], \"euclidean\")) / 2\n",
    "\n",
    "    return w, h\n",
    "\n",
    "\n",
    "def warpBox(image,\n",
    "            box,\n",
    "            target_height=None,\n",
    "            target_width=None,\n",
    "            margin=0,\n",
    "            cval=None,\n",
    "            return_transform=False):\n",
    "    \"\"\"Warp a boxed region in an image given by a set of four points into\n",
    "    a rectangle with a specified width and height. Useful for taking crops\n",
    "    of distorted or rotated text.\n",
    "\n",
    "    Args:\n",
    "        image: The image from which to take the box\n",
    "        box: A list of four points starting in the top left\n",
    "            corner and moving clockwise.\n",
    "        target_height: The height of the output rectangle\n",
    "        target_width: The width of the output rectangle\n",
    "        return_transform: Whether to return the transformation\n",
    "            matrix with the image.\n",
    "    \"\"\"\n",
    "\n",
    "    if cval is None:\n",
    "        cval = (0, 0, 0) if len(image.shape) == 3 else 0\n",
    "    box, _ = get_rotated_box(box)\n",
    "    w, h = get_rotated_width_height(box)\n",
    "    if h > 2*w:\n",
    "        min_x_idx = np.argmin(box[:,0])\n",
    "        min_x_p = box[min_x_idx]\n",
    "        max_x_idx = np.argmax(box[:,0])\n",
    "        max_x_p = box[max_x_idx]\n",
    "        if min_x_p[1] > max_x_p[1]:\n",
    "            n_roll = 1\n",
    "        else:\n",
    "            n_roll = 3\n",
    "        temp = h\n",
    "        h = w\n",
    "        w = temp\n",
    "        box = np.roll(box,n_roll,axis=0)\n",
    "        \n",
    "       \n",
    "    assert (\n",
    "            (target_width is None and target_height is None)\n",
    "            or (target_width is not None and target_height is not None)), \\\n",
    "        'Either both or neither of target width and height must be provided.'\n",
    "    if target_width is None and target_height is None:\n",
    "        target_width = w\n",
    "        target_height = h\n",
    "    scale = min(target_width / w, target_height / h)\n",
    "    M = cv2.getPerspectiveTransform(src=box,\n",
    "                                    dst=np.array([[margin, margin], [scale * w - margin, margin],\n",
    "                                                  [scale * w - margin, scale * h - margin],\n",
    "                                                  [margin, scale * h - margin]]).astype('float32'))\n",
    "    crop = cv2.warpPerspective(image, M, dsize=(int(scale * w), int(scale * h)))\n",
    "    target_shape = (target_height, target_width, 3) if len(image.shape) == 3 else (target_height,\n",
    "                                                                                   target_width)\n",
    "    full = (np.zeros(target_shape) + cval).astype('uint8')\n",
    "    full[:crop.shape[0], :crop.shape[1]] = crop\n",
    "    if return_transform:\n",
    "        return full, M\n",
    "    return full\n",
    "\n",
    "\n",
    "def get_rotated_box(points):\n",
    "    \"\"\"Obtain the parameters of a rotated box.\n",
    "\n",
    "    Returns:\n",
    "        The vertices of the rotated box in top-left,\n",
    "        top-right, bottom-right, bottom-left order along\n",
    "        with the angle of rotation about the bottom left corner.\n",
    "    \"\"\"\n",
    "    try:\n",
    "        mp = geometry.MultiPoint(points=points)\n",
    "        pts = np.array(list(zip(*mp.minimum_rotated_rectangle.exterior.xy)))[:-1]  # noqa: E501\n",
    "    except AttributeError:\n",
    "        # There weren't enough points for the minimum rotated rectangle function\n",
    "        pts = points\n",
    "    # The code below is taken from\n",
    "    # https://github.com/jrosebr1/imutils/blob/master/imutils/perspective.py\n",
    "\n",
    "    # sort the points based on their x-coordinates\n",
    "    xSorted = pts[np.argsort(pts[:, 0]), :]\n",
    "\n",
    "    # grab the left-most and right-most points from the sorted\n",
    "    # x-roodinate points\n",
    "    leftMost = xSorted[:2, :]\n",
    "    rightMost = xSorted[2:, :]\n",
    "\n",
    "    # now, sort the left-most coordinates according to their\n",
    "    # y-coordinates so we can grab the top-left and bottom-left\n",
    "    # points, respectively\n",
    "    leftMost = leftMost[np.argsort(leftMost[:, 1]), :]\n",
    "    (tl, bl) = leftMost\n",
    "\n",
    "    # now that we have the top-left coordinate, use it as an\n",
    "    # anchor to calculate the Euclidean distance between the\n",
    "    # top-left and right-most points; by the Pythagorean\n",
    "    # theorem, the point with the largest distance will be\n",
    "    # our bottom-right point\n",
    "    D = spatial.distance.cdist(tl[np.newaxis], rightMost, \"euclidean\")[0]\n",
    "    (br, tr) = rightMost[np.argsort(D)[::-1], :]\n",
    "\n",
    "    # return the coordinates in top-left, top-right,\n",
    "    # bottom-right, and bottom-left order\n",
    "    pts = np.array([tl, tr, br, bl], dtype=\"float32\")\n",
    "\n",
    "    rotation = np.arctan((tl[0] - bl[0]) / (tl[1] - bl[1]))\n",
    "    return pts, rotation\n",
    "\n",
    "\n",
    "def get_rect_rotation(polygon):\n",
    "    bbox = np.array(polygon.minimum_rotated_rectangle.exterior.coords).round().astype(np.int32)\n",
    "    pts = bbox[:4]\n",
    "     # sort the points based on their x-coordinates\n",
    "    xSorted = pts[np.argsort(pts[:, 0]), :]\n",
    "\n",
    "    # grab the left-most and right-most points from the sorted\n",
    "    # x-roodinate points\n",
    "    leftMost = xSorted[:2, :]\n",
    "    rightMost = xSorted[2:, :]\n",
    "\n",
    "    # now, sort the left-most coordinates according to their\n",
    "    # y-coordinates so we can grab the top-left and bottom-left\n",
    "    # points, respectively\n",
    "    leftMost = leftMost[np.argsort(leftMost[:, 1]), :]\n",
    "    (tl, bl) = leftMost\n",
    "\n",
    "    # now that we have the top-left coordinate, use it as an\n",
    "    # anchor to calculate the Euclidean distance between the\n",
    "    # top-left and right-most points; by the Pythagorean\n",
    "    # theorem, the point with the largest distance will be\n",
    "    # our bottom-right point\n",
    "    D = spatial.distance.cdist(tl[np.newaxis], rightMost, \"euclidean\")[0]\n",
    "    (br, tr) = rightMost[np.argsort(D)[::-1], :]\n",
    "\n",
    "    # return the coordinates in top-left, top-right,\n",
    "    # bottom-right, and bottom-left order\n",
    "    pts = np.array([tl, tr, br, bl], dtype=\"float32\")\n",
    "\n",
    "    rotation = np.arctan((tl[0] - bl[0]) / (tl[1] - bl[1]))\n",
    "    return rotation\n",
    "\n",
    "def get_nearby_boxes(box,ids_done,polygons,dist=0):\n",
    "    delta_degrees = 15\n",
    "    if dist > 0:\n",
    "        box = box.buffer(dist)\n",
    "    matches = set()\n",
    "    box_rotation = np.rad2deg(get_rect_rotation(box))\n",
    "    for i,p in enumerate(polygons):\n",
    "        if i in ids_done:\n",
    "            continue\n",
    "        # Search for intersecting boxes\n",
    "        if box.intersects(p):\n",
    "            # Check if their angles don't differ too much\n",
    "            rot = np.rad2deg(get_rect_rotation(p))\n",
    "            if abs(box_rotation - rot) <= delta_degrees:\n",
    "                matches.add(i)\n",
    "                \n",
    "    return matches\n",
    "\n",
    "\n",
    "def get_nearby_boxes_rec(box,skip_indexes,polygons,dist=0):\n",
    "    \n",
    "    total_matches = set()\n",
    "    # look for matches\n",
    "    matches = get_nearby_boxes(box,skip_indexes,polygons,dist)\n",
    "    total_matches |= matches\n",
    "    skip_indexes |= matches\n",
    "    while len(matches) > 0:\n",
    "        next_id = matches.pop()\n",
    "        next_box = polygons[next_id]\n",
    "        new_matches = get_nearby_boxes(next_box,skip_indexes,polygons,dist)\n",
    "        matches |= new_matches\n",
    "        total_matches |= matches\n",
    "        skip_indexes |= matches\n",
    "        \n",
    "    return total_matches\n",
    "\n",
    "# Sort left-to-right, top-to-bottom\n",
    "def sort_key(polygon):\n",
    "    c = list(polygon.centroid.coords)\n",
    "    \n",
    "    return (c[0][0]+c[0][1])/2\n",
    "\n",
    "\n",
    "def sort_polygons(polygons):\n",
    "    return sorted(polygons,key=sort_key)\n",
    "\n",
    "\n",
    "def sort_polygon_ids(polygons):\n",
    "    # if 2 or less polygons, we sort them as usual\n",
    "    if len(polygons) <= 2:\n",
    "        return np.argsort([sort_key(p) for p in polygons])\n",
    "    \n",
    "    # if there are more we sort top to bottom, split into groups\n",
    "    # then sort each group left to right\n",
    "    \n",
    "    # probably need to handle the case if text is oriented from bottom left to top right as this will not be correct!\n",
    "    centers = [list(p.centroid.coords) for p in polygons]\n",
    "    y_coords = np.array([c[0][1] for c in centers])\n",
    "    ids = np.arange(len(polygons))\n",
    "    sorted_ids = np.argsort(y_coords)\n",
    "    sorted_y = y_coords[sorted_ids]\n",
    "    ids = ids[sorted_ids]\n",
    "    curr_id = 0\n",
    "    delta_y = 10\n",
    "    groups = []\n",
    "   \n",
    "    curr_group = [sorted_ids[0]]\n",
    "    \n",
    "    for i in range(len(sorted_y)-1):\n",
    "        curr = sorted_y[i]\n",
    "        next_ = sorted_y[i+1]\n",
    "        if next_ - curr >= delta_y:\n",
    "            curr_id += 1\n",
    "            groups.append(curr_group)\n",
    "            curr_group = [sorted_ids[i+1]]\n",
    "        else:\n",
    "            curr_group.append(sorted_ids[i+1])\n",
    "        \n",
    "        if i == len(sorted_y)-2:\n",
    "            groups.append(curr_group)\n",
    "        \n",
    "\n",
    "    # flatten list\n",
    "    res_ids = [id_ for group in groups for id_ in group]\n",
    "    \n",
    "    return ids[res_ids]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Postprocess predictions\n",
    "\n",
    "#### Detections that have less than 3 chars detected: merge with overlapping ones and recognize again\n",
    "#### Detetions that have 3 or more chars: merge with overlapping, add spaces in reading direction\n",
    "\n",
    "#### Sort polygons, sorted in 2 steps: up/down => split in groups then sort each group left to right\n",
    "#### Merging overlapping detections: check if they are relatively the same direction, don't merge when angles differ 15 degrees or more\n",
    "#### Ignore detections with numbers in them when merging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current file: TOP50raster-31O-2018\n",
      "Current file: TOP50raster-32O-2018\n",
      "Current file: TOP50raster-32W-2018\n",
      "Current file: TOP50raster-38O-2018\n",
      "Current file: TOP50raster-39O-2018\n",
      "Current file: TOP50raster-39W-2018\n",
      "Current file: TOP50raster-44O-2018\n",
      "Current file: TOP50raster-45O-2018\n",
      "Current file: TOP50raster-45W-2018\n"
     ]
    }
   ],
   "source": [
    "for fname in os.listdir(img_dir):\n",
    "    if fname.endswith('.jpg'):\n",
    "        if '-Copy' in fname:\n",
    "            continue\n",
    "        map_name = fname.replace('.jpg','')\n",
    "        print('Current file:',map_name)\n",
    "        lines_fname = os.path.join(json_dir,fname.replace('.jpg','_lines.json'))\n",
    "        \n",
    "        # Check if the map has been fully processed before\n",
    "        if os.path.exists(lines_fname):\n",
    "            print('Map already processed')\n",
    "            continue\n",
    "        \n",
    "        words_fname = os.path.join(json_dir,map_name+'_words.json')\n",
    "        words = json.load(open(words_fname,'r'))['words']\n",
    "        img = cv2.imread(os.path.join(img_dir,fname))\n",
    "        \n",
    "        # merged\n",
    "        out = img.copy()\n",
    "        ids_done = set()\n",
    "        polygons = []\n",
    "        words_filtered = []\n",
    "        lines = []\n",
    "        # create polygons, ignore words with numbers, these should not be merged!\n",
    "        for w in words:\n",
    "            if any(char.isdigit() for char in w['text']):\n",
    "                w['recognize'] = 0\n",
    "                lines.append(w)\n",
    "            else:\n",
    "                words_filtered.append(w)\n",
    "                bbox = np.array(w['boundingBox'],dtype=np.int32).reshape((4,2))\n",
    "                polygons.append(geometry.Polygon(bbox))\n",
    "            \n",
    "            \n",
    "        for i,w in enumerate(words_filtered):\n",
    "            recognize = 0\n",
    "            text = w['text']\n",
    "            if i in ids_done:\n",
    "                continue\n",
    "            # look for nearby words\n",
    "            ids_done.add(i)\n",
    "            box = polygons[i]\n",
    "            matched_ids = get_nearby_boxes_rec(box,ids_done,polygons,dist=0)\n",
    "            ids_done |= matched_ids\n",
    "            matched_polygons = [box]\n",
    "            matched_text = [text]\n",
    "            if len(matched_ids) > 0:\n",
    "                \n",
    "                # filter out based on height   \n",
    "                for id_ in matched_ids:\n",
    "                    matched_polygons.append(polygons[id_])\n",
    "                    matched_text.append(words_filtered[id_]['text'])\n",
    "                \n",
    "                # Check length of words, if each word is >= 3 letters, merge\n",
    "                # Else take longest\n",
    "                if min(map(len,matched_text)) >= 3:\n",
    "                    # Sort polygons in reading direction\n",
    "                    sorted_ids = sort_polygon_ids(matched_polygons)\n",
    "                    # replace text\n",
    "                    matched_text = [matched_text[id_] for id_ in sorted_ids]\n",
    "                    text = ' '.join(matched_text)\n",
    "                    #print('Joined:',matched_text)\n",
    "                else:\n",
    "                    max_len_id = np.argmax([len(m) for m in matched_text])\n",
    "                    text = matched_text[max_len_id]\n",
    "                    recognize = 1\n",
    "                    #print('Took longest:',text,'from:',matched_text)\n",
    "        \n",
    "                multi = geometry.MultiPolygon(matched_polygons)\n",
    "            else:\n",
    "                multi = box\n",
    "            bbox = np.array(multi.minimum_rotated_rectangle.exterior.coords).round().astype(np.int32)\n",
    "            bbox = bbox[:4]\n",
    "            obj = {'text':text,'boundingBox':bbox.ravel().tolist(),'recognize':recognize}\n",
    "            lines.append(obj)\n",
    "        # uncomment to visualize\n",
    "        '''\n",
    "        for l in lines:\n",
    "            text = l['text']\n",
    "            bbox = np.array(l['boundingBox']).reshape((4,2))\n",
    "            x,y = bbox[0]-2\n",
    "            color = (0,225,0)\n",
    "            cv2.polylines(out, [bbox], True,color=color, thickness=2)\n",
    "            cv2.putText(out, text, (x,y), cv2.FONT_HERSHEY_SIMPLEX, 1, (225, 0, 0), 2, cv2.LINE_AA)\n",
    "        \n",
    "        cv2.imwrite(map_name+'_boxes.jpg',out)\n",
    "        '''\n",
    "        \n",
    "        json.dump({'lines':lines},open(words_fname.replace('_words.json','_lines.json'),'w'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "\n",
    "# advised to make a copy of the recognition results, in case of later errors\n",
    "lines_fnames = [os.path.join(json_dir,f) for f in os.listdir(json_dir) if '_lines' in f and '-Copy' not in f]\n",
    "for f in lines_fnames:\n",
    "    copy_path = f.replace('.json','-Copy1.json')\n",
    "    if not os.path.exists(copy_path):\n",
    "        shutil.copyfile(f,copy_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Words detection results have been postprocessed and merged\n",
    "\n",
    "#### Now we detect the merged results, adjusting for vertical orientation with warpBox. Words wich are vertical and detected, often have a very short length\n",
    "#### So for each line that has less than 3 chars detected or is vertical, we detect again, adjusted warpBox for vertical in both directions\n",
    "#### Because we cannot know which of the two strings is gibberish, we save them both and later filter with geocoder step\n",
    "#### Also run recognize again on small words that were merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking for pretrained_models/crnn_kurapan.h5\n",
      "Current file: TOP50raster-31O-2018\n",
      "Current file: TOP50raster-32O-2018\n",
      "Current file: TOP50raster-32W-2018\n",
      "Current file: TOP50raster-38O-2018\n",
      "Current file: TOP50raster-39O-2018\n",
      "Current file: TOP50raster-39W-2018\n",
      "Current file: TOP50raster-44O-2018\n",
      "Current file: TOP50raster-45O-2018\n",
      "Current file: TOP50raster-45W-2018\n"
     ]
    }
   ],
   "source": [
    "recognizer = keras_ocr.recognition.Recognizer()\n",
    "target_height = 31\n",
    "target_width = 200\n",
    "\n",
    "for fname in os.listdir(img_dir):\n",
    "    if '-Copy' in fname or '.ipy' in fname:\n",
    "        continue\n",
    "    map_name = fname.replace('.jpg','')\n",
    "    print('Current file:',map_name)\n",
    "    lines_fname = os.path.join(json_dir,map_name+'_lines.json')\n",
    "    lines = json.load(open(lines_fname,'r'))['lines']\n",
    "    skip = False\n",
    "    for line in lines:\n",
    "        if 'geolocation' in line:\n",
    "            skip = True\n",
    "            break\n",
    "    \n",
    "    if skip:\n",
    "        print('Map already processed')\n",
    "        continue\n",
    "    \n",
    "    img = keras_ocr.tools.read(os.path.join(img_dir,fname))\n",
    "\n",
    "    for line in lines:\n",
    "        text = line['text']\n",
    "        \n",
    "        if len(text) < 3 or line['recognize']:\n",
    "            bbox = np.array(line['boundingBox'],dtype=np.int32).reshape((4,2))\n",
    "            warped = warpBox(img,bbox,target_height=target_height,target_width=target_width)\n",
    "            bbox, _ = get_rotated_box(bbox)\n",
    "            w, h = get_rotated_width_height(bbox)\n",
    "            # Only run prediction again on vertical boxes or merged boxes from previous step\n",
    "            if line['recognize'] or h > 2*w:\n",
    "                # prediction on warpbox\n",
    "                prediction = recognizer.recognize(warped)\n",
    "\n",
    "                bbox, _ = get_rotated_box(bbox)\n",
    "                w, h = get_rotated_width_height(bbox)\n",
    "                \n",
    "                # If line is vertical, rotate it, predict again and add that prediction as well\n",
    "                if h > 2*w:\n",
    "                    inv = warped[::-1,::-1]\n",
    "                    inv_prediction = recognizer.recognize(inv)\n",
    "                    prediction = prediction.strip()+'|'+inv_prediction\n",
    "                    \n",
    "                line['text'] = prediction\n",
    "            \n",
    "            \n",
    "        json.dump({'lines':lines},open(lines_fname,'w'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Now perform geocoder requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def geocode_request_all(query, lat, lng,geonames_key=None,google_key=None,tomtom_key=None, radius=5000, min_score=50):\n",
    "    \"\"\"\n",
    "    Just returns all found values with all geocoders\n",
    "    \"\"\"\n",
    "\n",
    "    def merge(arr):\n",
    "        for g in arr:\n",
    "            name = g['name']\n",
    "            score = fuzz.partial_ratio(query, name.lower())\n",
    "            if score > min_score:\n",
    "                result.append((g, score))\n",
    "\n",
    "    \n",
    "    # query each geocoder\n",
    "    result = []\n",
    "    if geonames_key is not None:\n",
    "        geo_result = geonames_request(query,key=geonames_key, lat=lat, lng=lng, radius=radius)\n",
    "        merge(geo_result)\n",
    "    if google_key is not None:\n",
    "        google_result = google_geocode_request(query,key=google_key, lat=lat, lng=lng, radius=radius)\n",
    "        merge(google_result)\n",
    "    if tomtom_key is not None:\n",
    "        tomtom_result = tomtom_request(query,key=tomtom_key, lat=lat, lng=lng, radius=radius)\n",
    "        merge(tomtom_result)\n",
    "\n",
    "\n",
    "    # Sort results on score, take shortest length if there is a tie\n",
    "    result = sorted(result, key=lambda x: (x[1], -len(x[0]['name'])), reverse=True)\n",
    "    return result\n",
    "\n",
    "\n",
    "def geonames_request(query,key, lat=None, lng=None, radius=5000, max_rows=40):\n",
    "    query = urllib.parse.quote(query, safe='')\n",
    "    url = 'http://api.geonames.org/searchJSON?q=' + query + '&lang=local&country=NL&fuzzy=0.3&maxRows=' + \\\n",
    "    str(max_rows) + '&username=' + key\n",
    "    \n",
    "    if radius is not None:\n",
    "        radius = '&radius=' + str(radius / 1000)\n",
    "        url += radius\n",
    "    if lat is not None and lng is not None:\n",
    "        se, nw = get_bbox_from_radius(lat, lng, radius / 1000)\n",
    "        s = str(se[0])\n",
    "        e = str(se[1])\n",
    "        n = str(nw[0])\n",
    "        w = str(nw[1])\n",
    "        bounds = '&south=' + s + '&north=' + n + '&west=' + w + '&east=' + e\n",
    "        url += bounds\n",
    "\n",
    "    response = requests.get(url)\n",
    "\n",
    "    geonames_result = response.json()\n",
    "\n",
    "    result = []\n",
    "\n",
    "    duplicate = False\n",
    "    unique = {}\n",
    "\n",
    "    for i, g in enumerate(geonames_result['geonames']):\n",
    "\n",
    "        obj = {'lat': '', 'lng': '', 'name': '', 'type': ''}\n",
    "        type_ = ''\n",
    "        name = ''\n",
    "\n",
    "        if 'lat' in g:\n",
    "            obj['lat'] = float(g['lat'])\n",
    "        if 'lng' in g:\n",
    "            obj['lng'] = float(g['lng'])\n",
    "        # Prefer to use name for local\n",
    "        if 'name' in g:\n",
    "            name = g['name']\n",
    "            obj['name'] = name\n",
    "        if 'fcl' in g:\n",
    "            fcl = g['fcl']\n",
    "            if fcl == 'P':\n",
    "                type_ = 'locality'\n",
    "            elif fcl == 'A':\n",
    "                type_ = 'administrative_area'\n",
    "            elif fcl == 'H':\n",
    "                type_ = 'stream,lake'\n",
    "            elif fcl == 'R':\n",
    "                type_ = 'road'\n",
    "            elif fcl == 'L':\n",
    "                type_ = 'parks,area'\n",
    "            elif fcl == 'V':\n",
    "                type_ = 'forest'\n",
    "            elif fcl == 'T':\n",
    "                type_ = 'mountain,hill'\n",
    "            elif fcl == 'S':\n",
    "                type_ = 'spot,building,farm'\n",
    "            else:\n",
    "                type_ = fcl\n",
    "\n",
    "            obj['type'] = type_\n",
    "\n",
    "        if name in unique:\n",
    "            \n",
    "            idx = unique[name]\n",
    "            duplicate = result[idx]\n",
    "\n",
    "            if duplicate['type'] == 'administrative_area' and type_ == 'locality':\n",
    "                result[idx] = obj\n",
    "            '''\n",
    "            elif duplicate['type'] == 'locality' and type_ == 'administrative_area':\n",
    "                continue\n",
    "            '''\n",
    "        else:\n",
    "            unique[obj['name']] = i\n",
    "\n",
    "        result.append(obj)\n",
    "\n",
    "    return result\n",
    "\n",
    "\n",
    "def tomtom_request(query, key, lat=None, lng=None, radius=5000, idx_set=None):\n",
    "    \"\"\"\n",
    "    Fuzzy geocode query in a specified area\n",
    "    :param query:\n",
    "    :param idx_set:\n",
    "    :param lat:\n",
    "    :param lng:\n",
    "    :param radius:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    query = urllib.parse.quote(query, safe='')\n",
    "\n",
    "    url = 'https://api.tomtom.com/search/2/search/' + query + \\\n",
    "          '.json?typeahead=true&countrySet=NL&language=nl-NL&limit=100&minFuzzyLevel=1&maxFuzzyLevel=4&key' \\\n",
    "          '='+key\n",
    "\n",
    "    if idx_set is None:\n",
    "        idx_set = 'Geo,Addr,PAD,Str'\n",
    "\n",
    "    url += '&idxSet=' + idx_set\n",
    "\n",
    "    if lat is not None and lng is not None:\n",
    "        url += '&lat=' + str(lat) + '&lon=' + str(lng)\n",
    "    if radius is not None:\n",
    "        url += '&radius=' + str(radius)\n",
    "    try:\n",
    "        response = requests.get(url)\n",
    "        tomtom_result = response.json()\n",
    "    except BaseException as e:\n",
    "        print('ERROR TOMTOM REQUEST')\n",
    "        print('url', url)\n",
    "        print(e)\n",
    "        return []\n",
    "\n",
    "    res = []\n",
    "    try:\n",
    "        results = tomtom_result['results']\n",
    "    except KeyError:\n",
    "        print('KeyError TomTom')\n",
    "        return []\n",
    "    \n",
    "    def lower_first_letter(s):\n",
    "        return s[:1].lower() + s[1:] if s else ''\n",
    "\n",
    "    for r in results:\n",
    "        obj = {}\n",
    "        type_ = r['type']\n",
    "        addr = r['address']\n",
    "        p = r['position']\n",
    "        try:\n",
    "            if type_ == 'Geography':\n",
    "                entity_type = lower_first_letter(r['entityType'])\n",
    "                if entity_type in addr:\n",
    "                    name = addr[entity_type]\n",
    "                elif 'municipalitySubdivision' in addr:\n",
    "                    name = addr['municipalitySubdivision']\n",
    "                else:\n",
    "                    name = addr['freeformAddress']\n",
    "                    #print('freeformaddres')\n",
    "\n",
    "            elif type_ == 'Street' or type_ == 'Point Address' or type_ == 'Address Range':\n",
    "                name = addr['streetName']\n",
    "\n",
    "            else:\n",
    "                print('\\nTYPEEE', type_, '\\n')\n",
    "                name = 'Unknown Type - TomTom'\n",
    "                print(r)\n",
    "        except KeyError:\n",
    "            continue\n",
    "\n",
    "        if 'dist' in r:\n",
    "            obj['dist'] = r['dist']\n",
    "\n",
    "        obj['lat'] = float(p['lat'])\n",
    "        obj['lng'] = float(p['lon'])\n",
    "        obj['name'] = name\n",
    "        obj['type'] = type_\n",
    "\n",
    "        res.append(obj)\n",
    "\n",
    "    return res\n",
    "\n",
    "\n",
    "\n",
    "def google_geocode_request(query, key, lat=None, lng=None, radius=5000):\n",
    "    \"\"\"\n",
    "    Fuzzy geocoding query in a specified area\n",
    "    :param query:\n",
    "    :param lat:\n",
    "    :param lng:\n",
    "    :param radius:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "\n",
    "    query = urllib.parse.quote(query, safe='')\n",
    "    \n",
    "    url = 'https://maps.googleapis.com/maps/api/geocode/json?address=' + query + \\\n",
    "          '&language=nl&region=nl&key='+key\n",
    "\n",
    "\n",
    "    if lat is not None and lng is not None:\n",
    "        se, nw = get_bbox_from_radius(lat, lng, radius / 1000)\n",
    "        bounds = str(se[0]) + ',' + str(se[1]) + '|' + str(nw[0]) + ',' + str(nw[1])\n",
    "        url += '&bounds=' + bounds\n",
    "\n",
    "    response = requests.get(url)\n",
    "    results = []\n",
    "    google_result = response.json()\n",
    "\n",
    "    def check_int(n):\n",
    "        try:\n",
    "            n = int(n)\n",
    "            return True\n",
    "        except BaseException as e:\n",
    "            return False\n",
    "\n",
    "    for r in google_result['results']:\n",
    "        addr = r['address_components']\n",
    "        name = ''\n",
    "        type_ = ''\n",
    "        for a in addr:\n",
    "\n",
    "            # ignore house numbers as they are listed first\n",
    "            if a['types'] == ['street_number'] or check_int(a['long_name']):\n",
    "                continue\n",
    "\n",
    "            name = a['long_name']\n",
    "            type_ = ','.join(a['types'])\n",
    "            break\n",
    "\n",
    "        loc = r['geometry']['location']\n",
    "\n",
    "        obj = {'lat': loc['lat'], 'lng': loc['lng'], 'name': name, 'type': type_}\n",
    "        results.append(obj)\n",
    "\n",
    "    filtered = []\n",
    "    if lat is not None and lng is not None:\n",
    "        for r in results:\n",
    "            lat = r['lat']\n",
    "            lng = r['lng']\n",
    "\n",
    "            if lat < se[0] or lat > nw[0] or lng < nw[1] or lng > se[1]:\n",
    "                pass\n",
    "            else:\n",
    "                filtered.append(r)\n",
    "\n",
    "        return filtered\n",
    "    else:\n",
    "        return results\n",
    "\n",
    "# Radius in km\n",
    "def get_bbox_from_radius(lat_, lng_, radius):\n",
    "    \"\"\"\n",
    "    Make a bounding box given a coordinate and radius. The bbox will be the outscribed square\n",
    "    :param lat_:\n",
    "    :param lng_:\n",
    "    :param radius:\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    r = 6378.1\n",
    "    pi = math.pi\n",
    "    # length of diagonal of outscribed square\n",
    "    radius *= (math.sqrt(2) / 2)\n",
    "    # Southeast corner\n",
    "    brng = 135 * pi / 180\n",
    "    lat = lat_ * pi / 180\n",
    "    lng = lng_ * pi / 180\n",
    "\n",
    "    def get_lat_lng(brng, lat, lng):\n",
    "        # Do the math magic\n",
    "        lat = math.asin(math.sin(lat) * math.cos(radius / r) + math.cos(lat) * math.sin(radius / r) * math.cos(brng))\n",
    "        lng += math.atan2(math.sin(brng) * math.sin(radius / r) * math.cos(lat),\n",
    "                          math.cos(radius / r) - math.sin(lat) * math.sin(lat))\n",
    "\n",
    "        return lat * 180 / pi, lng * 180 / pi\n",
    "\n",
    "    lat1, lng1 = get_lat_lng(brng, lat, lng)\n",
    "\n",
    "    # Northwest corner\n",
    "    brng = 315 * pi / 180\n",
    "    lat = lat_ * pi / 180\n",
    "    lng = lng_ * pi / 180\n",
    "\n",
    "    lat2, lng2 = get_lat_lng(brng, lat, lng)\n",
    "\n",
    "    return (lat1, lng1), (lat2, lng2)\n",
    "\n",
    "\n",
    "def line_inside(bbox,line):\n",
    "    x_min,y_min,x_max,y_max = bbox\n",
    "    \n",
    "    bbox = line['boundingBox']\n",
    "    x = bbox[::2]\n",
    "    y = bbox[1::2]\n",
    "    x1 = min(x)\n",
    "    x2 = max(x)\n",
    "    y1 = min(y)\n",
    "    y2 = max(y)\n",
    "    \n",
    "    return x1 > x_min and x2 < x_max and y1 > y_min and y2 < y_max\n",
    "\n",
    "\n",
    "def haversine(lat1,lng1,lat2,lng2):\n",
    "\n",
    "    lng1, lat1, lng2, lat2 = map(math.radians, [lng1, lat1, lng2, lat2])\n",
    "    dlon = lng2 - lng1\n",
    "    dlat = lat2 - lat1\n",
    "    a = math.sin(dlat / 2) ** 2 + math.cos(lat1) * math.cos(lat2) * math.sin(dlon / 2) ** 2\n",
    "    return (2 * 6371 * math.asin(math.sqrt(a)))*1000\n",
    "\n",
    "\n",
    "def get_bbox_height(bbox):\n",
    "    x1,y1,x2,y2,x3,y3,x4,y4 = bbox\n",
    "    p1 = np.array([x1,y1])\n",
    "    p2 = np.array([x2,y2])\n",
    "    p3 = np.array([x3,y3])\n",
    "    p4 = np.array([x4,y4])\n",
    "    \n",
    "    d1 = np.sqrt(np.sum((p1-p4)**2))\n",
    "    d2 = np.sqrt(np.sum((p2-p3)**2))\n",
    "    \n",
    "    return (d1+d2)/2    \n",
    "\n",
    "\n",
    "def remove_duplicate_geolocations(geo):\n",
    "    placename_types = {'country','political','locality','administrative_area','sublocality','sublocality_level_1','geography'}\n",
    "    road_types = {'street','road','route'}\n",
    "    \n",
    "    for g_ in geo:\n",
    "        g = g_[0]\n",
    "        if 'type' in g:\n",
    "            typ = g['type']\n",
    "            types = typ.lower().split(',')\n",
    "            found=False\n",
    "            for t in types:\n",
    "                if t in placename_types:\n",
    "                    g['type'] = 'locality'\n",
    "                    found = True\n",
    "                    break\n",
    "                elif t in road_types:\n",
    "                    g['type'] = 'street'\n",
    "                    found = True\n",
    "                    break\n",
    "    \n",
    "    res = []\n",
    "    found_names = {}\n",
    "    # lat,lng\n",
    "    found_coords = np.array([[0,0]],dtype=np.float32)\n",
    "    delta_street = 0.01\n",
    "    delta_locality = 0.1\n",
    "    for g_ in geo:\n",
    "        g = g_[0]\n",
    "        if 'type' in g:\n",
    "            typ = g['type']\n",
    "            name = g['name'].lower()\n",
    "            lat = g['lat']\n",
    "            lng = g['lng']\n",
    "            c1 = np.array([lat,lng],dtype=np.float32)\n",
    "            \n",
    "            if name in found_names:\n",
    "                found_types = found_names[name]\n",
    "                # check if it is close\n",
    "                if typ in found_types:\n",
    "                   \n",
    "                    dlat =np.min(np.abs(c1[0]-found_coords[:,0]))\n",
    "                    dlng =np.min(np.abs(c1[1]-found_coords[:,1]))\n",
    "                    \n",
    "                    \n",
    "                    delta = delta_locality if typ == 'locality' else delta_street\n",
    "                    \n",
    "                    if dlat > delta and dlng > delta:\n",
    "                        res.append(g_)\n",
    "                        found_coords = np.append(found_coords,[c1],axis=0)\n",
    "                    \n",
    "                else:\n",
    "                    found_types.add(typ)\n",
    "                    res.append(g_)\n",
    "                    found_coords = np.append(found_coords,[c1],axis=0)\n",
    "                    \n",
    "            else:\n",
    "                found_names[name] = {typ}\n",
    "                res.append(g_)\n",
    "                found_coords = np.append(found_coords,[c1],axis=0)\n",
    "       \n",
    "    return res\n",
    "\n",
    "\n",
    "def filter_geo_inside(geolocations,lat_min,lng_min,lat_max,lng_max):\n",
    "    \"\"\"\n",
    "    Returns the geolocations which lie inside the supplied coordinates\n",
    "    \"\"\"\n",
    "    res =[]\n",
    "    for g in geolocations:\n",
    "        lat = g[0]['lat']\n",
    "        lng = g[0]['lng']\n",
    "        if lat_min < lat < lat_max and lng_min < lng < lng_max:\n",
    "            res.append(g)\n",
    "    return res\n",
    "\n",
    "def take_longest_match(text, geolocations):\n",
    "    \"\"\"\n",
    "    Filter out duplicate matches\n",
    "    e.g. if text = zandbergstr and matches are zandberg and zandbergstraat (both have 100 partial string similarity)\n",
    "    it will take zandbergstraat as that contains the longest substring\n",
    "    If both contain the same substring, both are returned\n",
    "    \"\"\"\n",
    "    #all have same score in arrary\n",
    "    if len(geolocations) < 2:\n",
    "        return geolocations\n",
    "    else:\n",
    "        idx_longest = -1\n",
    "        l_text = len(text)\n",
    "        # check differences in string length, take least diff if unique\n",
    "        d_lengths = [len(g[0]['name'])-l_text for g in geolocations]\n",
    "        abs_lengths = np.abs(d_lengths)\n",
    "        sorted_lengths = sorted(abs_lengths)\n",
    "        # if 2 or more best ones have same length diff\n",
    "        if sorted_lengths[0] == sorted_lengths[1]:\n",
    "            min_ = np.min([d if d >= 0 else l_text for d in d_lengths ])\n",
    "            return [geolocations[i] for i in range(len(d_lengths)) if d_lengths[i] == min_]\n",
    "        else:\n",
    "            min_idx = np.argmin(abs_lengths)\n",
    "            return [geolocations[min_idx]]\n",
    "\n",
    "        \n",
    "def haversine(lat1,lng1,lat2,lng2):\n",
    "\n",
    "    lng1, lat1, lng2, lat2 = map(math.radians, [lng1, lat1, lng2, lat2])\n",
    "    dlon = lng2 - lng1\n",
    "    dlat = lat2 - lat1\n",
    "    a = math.sin(dlat / 2) ** 2 + math.cos(lat1) * math.cos(lat2) * math.sin(dlon / 2) ** 2\n",
    "    return (2 * 6371 * math.asin(math.sqrt(a)))*1000        \n",
    "\n",
    "\n",
    "def overpass_bbox_query(lat1,lng1,lat2,lng2):\n",
    "    n = str(lat1)\n",
    "    w = str(lng1)\n",
    "    s = str(lat2)\n",
    "    e = str(lng2)\n",
    "    \n",
    "    args =  'data=[out:json];(way(' + n + ',' + w + ',' + s + ',' + e + ')[highway];);out%20geom;'\n",
    "    url = 'https://lz4.overpass-api.de/api/interpreter?' + args\n",
    "    \n",
    "    i = 1\n",
    "    while i < 4:\n",
    "        try:\n",
    "            res = requests.get(url).json()\n",
    "            return res\n",
    "        except BaseException as e:\n",
    "            print('Error with OSM query,retrying for the ' + str(i + 1) + 'th time')\n",
    "            print(e)\n",
    "            i += 1\n",
    "            if i < 4:\n",
    "                time.sleep(2)\n",
    "\n",
    "    return None\n",
    "\n",
    "\n",
    "def overpass_query(query,lat1,lng1,lat2,lng2):\n",
    "   \n",
    "    n = str(lat1)\n",
    "    w = str(lng1)\n",
    "    s = str(lat2)\n",
    "    e = str(lng2)\n",
    "    query = urllib.parse.quote(query, safe='')\n",
    "    args =  'data=[out:json];(way(' + n + ',' + w + ',' + s + ',' + e + ')[highway][name~\"^'+query+'\",i];);out%20geom;'\n",
    "    url = 'https://lz4.overpass-api.de/api/interpreter?' + args\n",
    "    \n",
    "    i = 1\n",
    "    while i < 4:\n",
    "        try:\n",
    "            res = requests.get(url).json()\n",
    "            return res\n",
    "        except BaseException as e:\n",
    "            print('Error with OSM query,retrying for the ' + str(i + 1) + 'th time')\n",
    "            print(e)\n",
    "            i += 1\n",
    "            if i < 4:\n",
    "                time.sleep(2)\n",
    "\n",
    "    return None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For vertical text, perform request for both orientations\n",
    "\n",
    "### Save queries and results in geocode_dict, so no queries are executed twice."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "geocode_fname = os.path.join(data_dir,'geocode_dict.json')\n",
    "geocode_dict = json.load(open(geocode_fname,'r',encoding='utf-8'))['dict']\n",
    "\n",
    "# Put your API key(s) here\n",
    "geonames_key = None\n",
    "google_key = None\n",
    "tomtom_key = None\n",
    "\n",
    "for fname in os.listdir(img_dir):\n",
    "    if '-Copy' in fname or '.ipy' in fname:\n",
    "        continue\n",
    "    map_name = fname.replace('.jpg','')\n",
    "    lines_fname = os.path.join(json_dir,map_name+'_lines.json')\n",
    "    lines = json.load(open(lines_fname,'r'))['lines']\n",
    "    print('Current file:',fname)\n",
    "    \n",
    "    for line in lines:\n",
    "       \n",
    "        \n",
    "        text = line['text']\n",
    "        if 'geolocation' in line:\n",
    "            continue\n",
    "        # Filter out numbers    \n",
    "        try:\n",
    "            z = int(text)\n",
    "            continue\n",
    "        except:\n",
    "            pass\n",
    "        \n",
    "        if len(text) > 3:\n",
    "            # If text was vertical and recognized twice\n",
    "            # split and perform 2 geocoder requests\n",
    "            if '|' in text:\n",
    "                t1 = text.split('|')[0]\n",
    "                t2 = text.split('|')[1]\n",
    "                if t1 not in geocode_dict:\n",
    "                    g1 = geocode_request_all(t1,geonames_key=geonames_key,google_key=google_key,tomtom_key=tomtom_key,\n",
    "                                             lat=None,lng=None)\n",
    "                    geocode_dict[t1] = g1\n",
    "                    time.sleep(0.2)\n",
    "                else:\n",
    "                    g1 = geocode_dict[t1]\n",
    "                    \n",
    "                if t2 not in geocode_dict:\n",
    "                    g2 = geocode_request_all(t2,geonames_key=geonames_key,google_key=google_key,tomtom_key=tomtom_key,\n",
    "                                             lat=None,lng=None)\n",
    "                    geocode_dict[t2] = g2\n",
    "                    time.sleep(0.2)\n",
    "                else:\n",
    "                    g2 = geocode_dict[t2]\n",
    "                    \n",
    "                line['geolocation'] = g1\n",
    "                line['geolocation2'] = g2\n",
    "            else:\n",
    "                if text not in geocode_dict:\n",
    "                    geo = geocode_request_all(text,geonames_key=geonames_key,google_key=google_key,tomtom_key=tomtom_key,\n",
    "                                              lat=None,lng=None)\n",
    "                    geocode_dict[text] = geo\n",
    "                    time.sleep(0.2)\n",
    "                else:\n",
    "                    geo = geocode_dict[text]\n",
    "                    \n",
    "                line['geolocation'] = geo\n",
    "                \n",
    "\n",
    "            json.dump({'dict':geocode_dict},open(geocode_fname,'w',encoding='utf-8'))\n",
    "            json.dump({'lines':lines},open(lines_fname,'w'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}