geoparsing_with_mordecai3_jo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "b8ggjAiOo9fP"
      ],
      "name": "geoparsing-with-mordecai3_jo.ipynb",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jonathanold/networks-in-conflict/blob/main/geoparsing_with_mordecai3_jo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PTKbbLvXn_Ob",
        "outputId": "df132022-130e-4a39-de21-a957a2abcb62",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: spacy==3.6.1 in /usr/local/lib/python3.10/dist-packages (3.6.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (8.1.12)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (1.1.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (2.0.10)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (0.9.0)\n",
            "Requirement already satisfied: pathy>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (0.11.0)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (6.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (4.66.2)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (1.25.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (2.31.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (2.6.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (3.1.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (23.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy==3.6.1) (3.3.0)\n",
            "Requirement already satisfied: pathlib-abc==0.1.1 in /usr/local/lib/python3.10/dist-packages (from pathy>=0.10.0->spacy==3.6.1) (0.1.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy==3.6.1) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.2 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy==3.6.1) (2.16.2)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy==3.6.1) (4.9.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy==3.6.1) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy==3.6.1) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy==3.6.1) (1.26.18)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy==3.6.1) (2024.2.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy==3.6.1) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy==3.6.1) (0.1.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy==3.6.1) (8.1.7)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy==3.6.1) (2.1.5)\n",
            "2024-02-26 18:52:25.292285: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
            "2024-02-26 18:52:25.292343: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
            "2024-02-26 18:52:25.293771: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2024-02-26 18:52:25.309236: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
            "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2024-02-26 18:52:27.011946: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n",
            "Collecting en-core-web-trf==3.6.1\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.6.1/en_core_web_trf-3.6.1-py3-none-any.whl (460.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m460.3/460.3 MB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: spacy<3.7.0,>=3.6.0 in /usr/local/lib/python3.10/dist-packages (from en-core-web-trf==3.6.1) (3.6.1)\n",
            "Requirement already satisfied: spacy-transformers<1.3.0,>=1.2.2 in /usr/local/lib/python3.10/dist-packages (from en-core-web-trf==3.6.1) (1.2.5)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (1.0.5)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (1.0.10)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (3.0.9)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (8.1.12)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (1.1.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.4.8)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.0.10)\n",
            "Requirement already satisfied: typer<0.10.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (0.9.0)\n",
            "Requirement already satisfied: pathy>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (0.11.0)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (6.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (4.66.2)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (1.25.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.31.0)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.6.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (3.1.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (23.2)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (3.3.0)\n",
            "Requirement already satisfied: transformers<4.31.0,>=3.4.0 in /usr/local/lib/python3.10/dist-packages (from spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (4.30.2)\n",
            "Requirement already satisfied: torch>=1.8.0 in /usr/local/lib/python3.10/dist-packages (from spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (1.13.1)\n",
            "Requirement already satisfied: spacy-alignments<1.0.0,>=0.7.2 in /usr/local/lib/python3.10/dist-packages (from spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (0.9.1)\n",
            "Requirement already satisfied: pathlib-abc==0.1.1 in /usr/local/lib/python3.10/dist-packages (from pathy>=0.10.0->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (0.1.1)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (0.6.0)\n",
            "Requirement already satisfied: pydantic-core==2.16.2 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.16.2)\n",
            "Requirement already satisfied: typing-extensions>=4.6.1 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (4.9.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (1.26.18)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2024.2.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (0.7.11)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (0.1.4)\n",
            "Requirement already satisfied: nvidia-cuda-runtime-cu11==11.7.99 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (11.7.99)\n",
            "Requirement already satisfied: nvidia-cudnn-cu11==8.5.0.96 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (8.5.0.96)\n",
            "Requirement already satisfied: nvidia-cublas-cu11==11.10.3.66 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (11.10.3.66)\n",
            "Requirement already satisfied: nvidia-cuda-nvrtc-cu11==11.7.99 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (11.7.99)\n",
            "Requirement already satisfied: wheel in /usr/local/lib/python3.10/dist-packages (from nvidia-cublas-cu11==11.10.3.66->torch>=1.8.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (0.42.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers<4.31.0,>=3.4.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers<4.31.0,>=3.4.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (0.20.3)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers<4.31.0,>=3.4.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers<4.31.0,>=3.4.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (2023.12.25)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers<4.31.0,>=3.4.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (0.13.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers<4.31.0,>=3.4.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (0.4.2)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.10.0,>=0.3.0->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (8.1.7)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy<3.7.0,>=3.6.0->en-core-web-trf==3.6.1) (2.1.5)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers<4.31.0,>=3.4.0->spacy-transformers<1.3.0,>=1.2.2->en-core-web-trf==3.6.1) (2023.6.0)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_trf')\n"
          ]
        }
      ],
      "source": [
        "# %%capture\n",
        "# Install dependencies\n",
        "\n",
        "# !pip install elasticsearch==7.14.0\n",
        "# !apt install default-jdk > /dev/null\n",
        "# !pip install -q mordecai3\n",
        "!pip install spacy==3.6.1\n",
        "!python -m spacy download en_core_web_trf"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Download & extract Elasticsearch 7.13.0\n",
        "\n",
        "!wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.13.0-linux-x86_64.tar.gz -q\n",
        "!tar -xzf elasticsearch-7.13.0-linux-x86_64.tar.gz\n",
        "!chown -R daemon:daemon elasticsearch-7.13.0"
      ],
      "metadata": {
        "id": "bnTe63UGurhY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Use prebuilt indexes\n",
        "Faster, but will not stay up to date."
      ],
      "metadata": {
        "id": "VPrcA9cNpFvY"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "# Download & extract es-geonames prebuilt indexes\n",
        "\n",
        "!gdown https://drive.google.com/uc?id=1AVR9NKi0NYGT6jKS4csl4B_W2Xktb_Rr\n",
        "!tar -xzvf geonames_index.tar.gz -C /content/elasticsearch-7.13.0/"
      ],
      "metadata": {
        "id": "SkmzVXnh5Ljr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%bash --bg\n",
        "sudo -H -u daemon elasticsearch-7.13.0/bin/elasticsearch"
      ],
      "metadata": {
        "id": "xoaaTYawoX5h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Sleep for few seconds to let the instance start.\n",
        "import time\n",
        "time.sleep(30)"
      ],
      "metadata": {
        "id": "jMym1K17p1Hr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%bash\n",
        "echo \"Change disk availability limits...\"\n",
        "curl -X PUT \"localhost:9200/_cluster/settings\" -H 'Content-Type: application/json' -d'\n",
        "{\n",
        "  \"transient\": {\n",
        "    \"cluster.routing.allocation.disk.watermark.low\": \"10gb\",\n",
        "    \"cluster.routing.allocation.disk.watermark.high\": \"5gb\",\n",
        "    \"cluster.routing.allocation.disk.watermark.flood_stage\": \"4gb\",\n",
        "    \"cluster.info.update.interval\": \"1m\"\n",
        "  }\n",
        "}\n",
        "'\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MdJ4LkWQvShq",
        "outputId": "d1b45b31-ed9b-480a-82dd-7eba535a55a1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Change disk availability limits...\n",
            "{\"acknowledged\":true,\"persistent\":{},\"transient\":{\"cluster\":{\"routing\":{\"allocation\":{\"disk\":{\"watermark\":{\"low\":\"10gb\",\"flood_stage\":\"4gb\",\"high\":\"5gb\"}}}},\"info\":{\"update\":{\"interval\":\"1m\"}}}}}"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "\r  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0\r100   452  100   195  100   257   1217   1605 --:--:-- --:--:-- --:--:--  2842\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Build ES-Geonames indexes\n",
        "Very slow (approx. 40 min.), but latest indexes."
      ],
      "metadata": {
        "id": "b8ggjAiOo9fP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "!pip install -q textacy"
      ],
      "metadata": {
        "id": "tXaLwiS5psWc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%bash --bg\n",
        "sudo -H -u daemon elasticsearch-7.13.0/bin/elasticsearch"
      ],
      "metadata": {
        "id": "MUf9boxi7c_I"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Sleep for few seconds to let the instance start.\n",
        "import time\n",
        "time.sleep(30)"
      ],
      "metadata": {
        "id": "gqnpHHHO7dL9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%writefile geonames_mapping.json\n",
        "{\n",
        "    \"settings\" : {\n",
        "        \"number_of_shards\" : 1,\n",
        "        \"number_of_replicas\" : 1\n",
        "    },\n",
        "    \"mappings\" : {\n",
        "        \"properties\" : {\n",
        "          \"geonameid\" : {\"type\" : \"keyword\", \"index\": \"true\"},\n",
        "          \"name\" : {\"type\" : \"text\"},\n",
        "          \"asciiname\" : {\"type\" : \"text\"},\n",
        "          \"alternativenames\" : {\"type\" : \"text\", \"similarity\" : \"boolean\",\n",
        "                                \"norms\": false},\n",
        "          \"coordinates\" : {\"type\" : \"geo_point\"},\n",
        "          \"feature_class\" :  {\"type\" : \"keyword\", \"index\":  \"true\"},\n",
        "          \"feature_code\" : {\"type\" : \"keyword\", \"index\":    \"true\"},\n",
        "          \"country_code3\" : {\"type\" : \"keyword\", \"index\":   \"true\"},\n",
        "          \"admin1_code\" : {\"type\" : \"keyword\", \"index\":  \"true\"},\n",
        "          \"admin1_name\" : {\"type\" : \"keyword\", \"index\":  \"true\"},\n",
        "          \"admin2_code\" : {\"type\" : \"keyword\", \"index\":  \"true\"},\n",
        "          \"admin2_name\" : {\"type\" : \"keyword\", \"index\":  \"true\"},\n",
        "          \"admin3_code\" : {\"type\" : \"keyword\", \"index\":  \"true\"},\n",
        "          \"admin4_code\" : {\"type\" : \"keyword\", \"index\":  \"true\"},\n",
        "          \"population\" : {\"type\" :  \"long\"},\n",
        "          \"alt_name_length\": {\"type\": \"long\"},\n",
        "          \"modification_date\" : {\"type\" : \"date\", \"format\": \"date\"}\n",
        "          }\n",
        "       }\n",
        "}"
      ],
      "metadata": {
        "id": "VRKWDbSBpa0i"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%bash\n",
        "echo \"Starting Docker container and data volume...\"\n",
        "# create the directory first to avoid permission issues when Docker is running as root\n",
        "mkdir $PWD/geonames_index/\n",
        "docker run -d -p 127.0.0.1:9200:9200 -e \"discovery.type=single-node\" -v $PWD/geonames_index/:/usr/share/elasticsearch/data elasticsearch:7.10.1\n",
        "\n",
        "echo \"Downloading Geonames gazetteer...\"\n",
        "wget https://download.geonames.org/export/dump/allCountries.zip\n",
        "wget https://download.geonames.org/export/dump/admin1CodesASCII.txt\n",
        "wget https://download.geonames.org/export/dump/admin2Codes.txt\n",
        "echo \"Unpacking Geonames gazetteer...\"\n",
        "unzip allCountries.zip\n",
        "\n",
        "echo \"Creating mappings for the fields in the Geonames index...\"\n",
        "curl -XPUT 'localhost:9200/geonames' -H 'Content-Type: application/json' -d @geonames_mapping.json\n",
        "\n",
        "echo \"Change disk availability limits...\"\n",
        "curl -X PUT \"localhost:9200/_cluster/settings\" -H 'Content-Type: application/json' -d'\n",
        "{\n",
        "  \"transient\": {\n",
        "    \"cluster.routing.allocation.disk.watermark.low\": \"10gb\",\n",
        "    \"cluster.routing.allocation.disk.watermark.high\": \"5gb\",\n",
        "    \"cluster.routing.allocation.disk.watermark.flood_stage\": \"4gb\",\n",
        "    \"cluster.info.update.interval\": \"1m\"\n",
        "  }\n",
        "}\n",
        "'\n"
      ],
      "metadata": {
        "id": "WdllUDrjpfQM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from elasticsearch import Elasticsearch, helpers\n",
        "import csv\n",
        "import sys\n",
        "from datetime import datetime\n",
        "from tqdm import tqdm\n",
        "import time\n",
        "from textacy.preprocessing.remove import accents as remove_accents\n",
        "\n",
        "#csv.field_size_limit(sys.maxsize)\n",
        "csv.field_size_limit()\n",
        "es = Elasticsearch(urls='http://localhost:9200/', timeout=60, max_retries=2)\n",
        "\n",
        "\n",
        "def iso_convert(iso2c):\n",
        "    \"\"\"\n",
        "    Takes a two character ISO country code and returns the corresponding 3\n",
        "    character ISO country code.\n",
        "    Parameters\n",
        "    ----------\n",
        "    iso2c: A two character ISO country code.\n",
        "    Returns\n",
        "    -------\n",
        "    iso3c: A three character ISO country code.\n",
        "    \"\"\"\n",
        "\n",
        "    iso_dict = {\"AD\":\"AND\", \"AE\":\"ARE\", \"AF\":\"AFG\", \"AG\":\"ATG\", \"AI\":\"AIA\",\n",
        "                \"AL\":\"ALB\", \"AM\":\"ARM\", \"AO\":\"AGO\", \"AQ\":\"ATA\", \"AR\":\"ARG\",\n",
        "                \"AS\":\"ASM\", \"AT\":\"AUT\", \"AU\":\"AUS\", \"AW\":\"ABW\", \"AX\":\"ALA\",\n",
        "                \"AZ\":\"AZE\", \"BA\":\"BIH\", \"BB\":\"BRB\", \"BD\":\"BGD\", \"BE\":\"BEL\",\n",
        "                \"BF\":\"BFA\", \"BG\":\"BGR\", \"BH\":\"BHR\", \"BI\":\"BDI\", \"BJ\":\"BEN\",\n",
        "                \"BL\":\"BLM\", \"BM\":\"BMU\", \"BN\":\"BRN\", \"BO\":\"BOL\", \"BQ\":\"BES\",\n",
        "                \"BR\":\"BRA\", \"BS\":\"BHS\", \"BT\":\"BTN\", \"BV\":\"BVT\", \"BW\":\"BWA\",\n",
        "                \"BY\":\"BLR\", \"BZ\":\"BLZ\", \"CA\":\"CAN\", \"CC\":\"CCK\", \"CD\":\"COD\",\n",
        "                \"CF\":\"CAF\", \"CG\":\"COG\", \"CH\":\"CHE\", \"CI\":\"CIV\", \"CK\":\"COK\",\n",
        "                \"CL\":\"CHL\", \"CM\":\"CMR\", \"CN\":\"CHN\", \"CO\":\"COL\", \"CR\":\"CRI\",\n",
        "                \"CU\":\"CUB\", \"CV\":\"CPV\", \"CW\":\"CUW\", \"CX\":\"CXR\", \"CY\":\"CYP\",\n",
        "                \"CZ\":\"CZE\", \"DE\":\"DEU\", \"DJ\":\"DJI\", \"DK\":\"DNK\", \"DM\":\"DMA\",\n",
        "                \"DO\":\"DOM\", \"DZ\":\"DZA\", \"EC\":\"ECU\", \"EE\":\"EST\", \"EG\":\"EGY\",\n",
        "                \"EH\":\"ESH\", \"ER\":\"ERI\", \"ES\":\"ESP\", \"ET\":\"ETH\", \"FI\":\"FIN\",\n",
        "                \"FJ\":\"FJI\", \"FK\":\"FLK\", \"FM\":\"FSM\", \"FO\":\"FRO\", \"FR\":\"FRA\",\n",
        "                \"GA\":\"GAB\", \"GB\":\"GBR\", \"GD\":\"GRD\", \"GE\":\"GEO\", \"GF\":\"GUF\",\n",
        "                \"GG\":\"GGY\", \"GH\":\"GHA\", \"GI\":\"GIB\", \"GL\":\"GRL\", \"GM\":\"GMB\",\n",
        "                \"GN\":\"GIN\", \"GP\":\"GLP\", \"GQ\":\"GNQ\", \"GR\":\"GRC\", \"GS\":\"SGS\",\n",
        "                \"GT\":\"GTM\", \"GU\":\"GUM\", \"GW\":\"GNB\", \"GY\":\"GUY\", \"HK\":\"HKG\",\n",
        "                \"HM\":\"HMD\", \"HN\":\"HND\", \"HR\":\"HRV\", \"HT\":\"HTI\", \"HU\":\"HUN\",\n",
        "                \"ID\":\"IDN\", \"IE\":\"IRL\", \"IL\":\"ISR\", \"IM\":\"IMN\", \"IN\":\"IND\",\n",
        "                \"IO\":\"IOT\", \"IQ\":\"IRQ\", \"IR\":\"IRN\", \"IS\":\"ISL\", \"IT\":\"ITA\",\n",
        "                \"JE\":\"JEY\", \"JM\":\"JAM\", \"JO\":\"JOR\", \"JP\":\"JPN\", \"KE\":\"KEN\",\n",
        "                \"KG\":\"KGZ\", \"KH\":\"KHM\", \"KI\":\"KIR\", \"KM\":\"COM\", \"KN\":\"KNA\",\n",
        "                \"KP\":\"PRK\", \"KR\":\"KOR\", \"XK\":\"XKX\", \"KW\":\"KWT\", \"KY\":\"CYM\",\n",
        "                \"KZ\":\"KAZ\", \"LA\":\"LAO\", \"LB\":\"LBN\", \"LC\":\"LCA\", \"LI\":\"LIE\",\n",
        "                \"LK\":\"LKA\", \"LR\":\"LBR\", \"LS\":\"LSO\", \"LT\":\"LTU\", \"LU\":\"LUX\",\n",
        "                \"LV\":\"LVA\", \"LY\":\"LBY\", \"MA\":\"MAR\", \"MC\":\"MCO\", \"MD\":\"MDA\",\n",
        "                \"ME\":\"MNE\", \"MF\":\"MAF\", \"MG\":\"MDG\", \"MH\":\"MHL\", \"MK\":\"MKD\",\n",
        "                \"ML\":\"MLI\", \"MM\":\"MMR\", \"MN\":\"MNG\", \"MO\":\"MAC\", \"MP\":\"MNP\",\n",
        "                \"MQ\":\"MTQ\", \"MR\":\"MRT\", \"MS\":\"MSR\", \"MT\":\"MLT\", \"MU\":\"MUS\",\n",
        "                \"MV\":\"MDV\", \"MW\":\"MWI\", \"MX\":\"MEX\", \"MY\":\"MYS\", \"MZ\":\"MOZ\",\n",
        "                \"NA\":\"NAM\", \"NC\":\"NCL\", \"NE\":\"NER\", \"NF\":\"NFK\", \"NG\":\"NGA\",\n",
        "                \"NI\":\"NIC\", \"NL\":\"NLD\", \"NO\":\"NOR\", \"NP\":\"NPL\", \"NR\":\"NRU\",\n",
        "                \"NU\":\"NIU\", \"NZ\":\"NZL\", \"OM\":\"OMN\", \"PA\":\"PAN\", \"PE\":\"PER\",\n",
        "                \"PF\":\"PYF\", \"PG\":\"PNG\", \"PH\":\"PHL\", \"PK\":\"PAK\", \"PL\":\"POL\",\n",
        "                \"PM\":\"SPM\", \"PN\":\"PCN\", \"PR\":\"PRI\", \"PS\":\"PSE\", \"PT\":\"PRT\",\n",
        "                \"PW\":\"PLW\", \"PY\":\"PRY\", \"QA\":\"QAT\", \"RE\":\"REU\", \"RO\":\"ROU\",\n",
        "                \"RS\":\"SRB\", \"RU\":\"RUS\", \"RW\":\"RWA\", \"SA\":\"SAU\", \"SB\":\"SLB\",\n",
        "                \"SC\":\"SYC\", \"SD\":\"SDN\", \"SS\":\"SSD\", \"SE\":\"SWE\", \"SG\":\"SGP\",\n",
        "                \"SH\":\"SHN\", \"SI\":\"SVN\", \"SJ\":\"SJM\", \"SK\":\"SVK\", \"SL\":\"SLE\",\n",
        "                \"SM\":\"SMR\", \"SN\":\"SEN\", \"SO\":\"SOM\", \"SR\":\"SUR\", \"ST\":\"STP\",\n",
        "                \"SV\":\"SLV\", \"SX\":\"SXM\", \"SY\":\"SYR\", \"SZ\":\"SWZ\", \"TC\":\"TCA\",\n",
        "                \"TD\":\"TCD\", \"TF\":\"ATF\", \"TG\":\"TGO\", \"TH\":\"THA\", \"TJ\":\"TJK\",\n",
        "                \"TK\":\"TKL\", \"TL\":\"TLS\", \"TM\":\"TKM\", \"TN\":\"TUN\", \"TO\":\"TON\",\n",
        "                \"TR\":\"TUR\", \"TT\":\"TTO\", \"TV\":\"TUV\", \"TW\":\"TWN\", \"TZ\":\"TZA\",\n",
        "                \"UA\":\"UKR\", \"UG\":\"UGA\", \"UM\":\"UMI\", \"US\":\"USA\", \"UY\":\"URY\",\n",
        "                \"UZ\":\"UZB\", \"VA\":\"VAT\", \"VC\":\"VCT\", \"VE\":\"VEN\", \"VG\":\"VGB\",\n",
        "                \"VI\":\"VIR\", \"VN\":\"VNM\", \"VU\":\"VUT\", \"WF\":\"WLF\", \"WS\":\"WSM\",\n",
        "                \"YE\":\"YEM\", \"YT\":\"MYT\", \"ZA\":\"ZAF\", \"ZM\":\"ZMB\", \"ZW\":\"ZWE\",\n",
        "                \"CS\":\"SCG\", \"AN\":\"ANT\"}\n",
        "\n",
        "    try:\n",
        "        iso3c = iso_dict[iso2c]\n",
        "        return iso3c\n",
        "    except KeyError:\n",
        "        print('Bad code: ' + iso2c)\n",
        "        iso3c = \"NA\"\n",
        "        return iso3c\n",
        "\n",
        "\n",
        "def read_adm1(fn=\"admin1CodesASCII.txt\"):\n",
        "    adm1_dict = {}\n",
        "    with open(fn, 'rt', encoding='utf-8') as f:\n",
        "        reader = csv.reader(f, delimiter='\\t')\n",
        "        for row in reader:\n",
        "            adm1_dict[row[0]] = row[1]\n",
        "    return adm1_dict\n",
        "\n",
        "\n",
        "def read_adm2(fn=\"admin2Codes.txt\"):\n",
        "    adm2_dict = {}\n",
        "    with open(fn, 'rt', encoding='utf-8') as f:\n",
        "        reader = csv.reader(f, delimiter='\\t')\n",
        "        for row in reader:\n",
        "            adm2_dict[row[0]] = row[1]\n",
        "    return adm2_dict\n",
        "\n",
        "def documents(reader, adm1_dict, adm2_dict, expand_ascii=True):\n",
        "    \"\"\"\n",
        "    Load Geonames entries provided by the `reader` into Elasticsearch.\n",
        "\n",
        "    If `expand_ascii` = True, any alternative names with accents will have an\n",
        "    accent-stripped form included in the alternative names list along with the original form.\n",
        "    For example, the name \"Ḩadīqat ash Shahbā\" would generate a second entry\n",
        "    \"Hadiqat ash Shahba\". This process does not affect non-Roman characters\n",
        "    (e.g. \"北京\" remains \"北京\".)\n",
        "\n",
        "    Parameters\n",
        "    ----------\n",
        "    reader: a CSV reader object\n",
        "    adm1_dict: dict\n",
        "      map from numeric ADM1 codes to names\n",
        "    adm2_dict: dict\n",
        "      map from numeric ADM2 codes to names\n",
        "    expand_ascii: bool\n",
        "      Include versions of names with accents stripped out.\n",
        "    \"\"\"\n",
        "    todays_date = datetime.today().strftime(\"%Y-%m-%d\")\n",
        "    count = 0\n",
        "    for row in tqdm(reader, total=12237435): # approx\n",
        "        try:\n",
        "            coords = row[4] + \",\" + row[5]\n",
        "            country_code3 = iso_convert(row[8])\n",
        "            alt_names = list(set(row[3].split(\",\")))\n",
        "            # so annoying...add \"US\" as an alt name for USA\n",
        "            if str(row[0]) == \"6252001\":\n",
        "                alt_names.append(\"US\")\n",
        "                alt_names.append(\"U.S.\")\n",
        "            if str(row[0]) == \"239880\":\n",
        "                alt_names.append(\"C.A.R.\")\n",
        "            alt_name_length = len(alt_names)\n",
        "            if expand_ascii:\n",
        "                stripped = [remove_accents(i) for i in alt_names]\n",
        "                both = alt_names + stripped\n",
        "                both = list(set(both))\n",
        "                alt_names = both\n",
        "            # get ADM1 name\n",
        "            if row[10]:\n",
        "                country_admin1 = '.'.join([row[8], row[10]])\n",
        "                try:\n",
        "                    admin1_name = adm1_dict[country_admin1]\n",
        "                except KeyError:\n",
        "                    admin1_name = \"\"\n",
        "            else:\n",
        "                admin1_name = \"\"\n",
        "            # Get ADM2 name\n",
        "            if row[11]:\n",
        "                country_admin2 = '.'.join([row[8], row[10], row[11]])\n",
        "                try:\n",
        "                    admin2_name = adm2_dict[country_admin2]\n",
        "                except KeyError:\n",
        "                    admin2_name = \"\"\n",
        "                    count += 1\n",
        "            else:\n",
        "                admin2_name = \"\"\n",
        "            doc = {\"geonameid\" : row[0],\n",
        "                    \"name\" : row[1],\n",
        "                    \"asciiname\" : row[2],\n",
        "                    \"alternativenames\" : alt_names,\n",
        "                    \"coordinates\" : coords,  # 4, 5\n",
        "                    \"feature_class\" : row[6],\n",
        "                    \"feature_code\" : row[7],\n",
        "                    \"country_code3\" : country_code3,\n",
        "                    \"admin1_code\" : row[10],\n",
        "                    \"admin1_name\": admin1_name,\n",
        "                    \"admin2_code\" : row[11],\n",
        "                    \"admin2_name\": admin2_name,\n",
        "                    \"admin3_code\" : row[12],\n",
        "                    \"admin4_code\" : row[13],\n",
        "                    \"population\" : row[14],\n",
        "                    \"alt_name_length\": alt_name_length,\n",
        "                    \"modification_date\" : todays_date\n",
        "                   }\n",
        "            action = {\"_index\" : \"geonames\",\n",
        "                      \"_id\" : doc['geonameid'],\n",
        "                      \"_source\" : doc}\n",
        "            yield action\n",
        "        except Exception as e:\n",
        "            print(e, row)\n",
        "            count += 1\n",
        "    print('Exception count:', count)\n",
        "\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    t = time.time()\n",
        "    adm1_dict = read_adm1()\n",
        "    adm2_dict = read_adm2()\n",
        "    f = open('allCountries.txt', 'rt', encoding='utf-8')\n",
        "    reader = csv.reader(f, delimiter='\\t')\n",
        "    actions = documents(reader, adm1_dict, adm2_dict)\n",
        "    helpers.bulk(es, actions, chunk_size=500)\n",
        "    es.indices.refresh(index='geonames')\n",
        "    e = (time.time() - t) / 60\n",
        "    print(\"Elapsed minutes: \", e)\n"
      ],
      "metadata": {
        "id": "Vcy57fklpnTQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Geoparser inference"
      ],
      "metadata": {
        "id": "ChgJC7lHom8r"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install --upgrade transformers==4.30.2\n",
        "from mordecai3 import Geoparser\n",
        "geo = Geoparser()"
      ],
      "metadata": {
        "id": "CAyvKbMOopTl",
        "outputId": "15077949-78ce-4cc5-88f7-c4cec37a62fe",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: transformers==4.30.2 in /usr/local/lib/python3.10/dist-packages (4.30.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.2) (3.13.1)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.14.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.2) (0.20.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.2) (1.25.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.2) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.2) (6.0.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.2) (2023.12.25)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.2) (2.31.0)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.2) (0.13.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.2) (0.4.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.30.2) (4.66.2)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.30.2) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.14.1->transformers==4.30.2) (4.9.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.30.2) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.30.2) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.30.2) (1.26.18)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.30.2) (2024.2.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Get file\n",
        "from google.colab import files\n",
        "uploaded = files.upload()\n",
        "import io\n",
        "data_satp = pd.read_csv(io.BytesIO(uploaded['dataset_satp_2019.csv']))\n",
        "# Dataset is now stored in a Pandas Dataframe\n",
        "events_2019 = data_satp['description'].tolist()\n"
      ],
      "metadata": {
        "id": "GrGGNLBbvnUB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "combined_df = pd.DataFrame()\n",
        "\n",
        "for z in range(0,len(events_2019)):\n",
        "  b = geo.geoparse_doc(events_2019[z])\n",
        "  # Extract the 'geolocated_ents' list and normalize it\n",
        "  geolocated_ents_data = b.get('geolocated_ents', [])\n",
        "  df = pd.json_normalize(geolocated_ents_data)\n",
        "\n",
        "  # Combine the flattened data with the non-nested columns\n",
        "  df = pd.concat([pd.DataFrame(b).drop('geolocated_ents', axis=1), df], axis=1)\n",
        "  combined_df = combined_df.append(df, ignore_index=True)\n",
        "\n",
        "\n",
        "\n",
        "b"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lddVc_ByoqMZ",
        "outputId": "476b4393-dd75-47e8-9125-7d83458bd663"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/elasticsearch/connection/base.py:208: ElasticsearchWarning: Elasticsearch built-in security features are not enabled. Without authentication, your cluster could be accessible to anyone. See https://www.elastic.co/guide/en/elasticsearch/reference/7.13/security-minimal-setup.html to enable security.\n",
            "  warnings.warn(message, category=ElasticsearchWarning)\n",
            "INFO:elasticsearch:POST http://localhost:9200/geonames/_search [status:200 request:0.054s]\n",
            "2024-02-26 21:00:40,666 elasticsearch INFO     POST http://localhost:9200/geonames/_search [status:200 request:0.054s]\n",
            "<ipython-input-16-07c326d2e514>:14: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
            "  combined_df = combined_df.append(df, ignore_index=True)\n",
            "/usr/local/lib/python3.10/dist-packages/elasticsearch/connection/base.py:208: ElasticsearchWarning: Elasticsearch built-in security features are not enabled. Without authentication, your cluster could be accessible to anyone. See https://www.elastic.co/guide/en/elasticsearch/reference/7.13/security-minimal-setup.html to enable security.\n",
            "  warnings.warn(message, category=ElasticsearchWarning)\n",
            "INFO:elasticsearch:POST http://localhost:9200/geonames/_search [status:200 request:0.017s]\n",
            "2024-02-26 21:00:41,016 elasticsearch INFO     POST http://localhost:9200/geonames/_search [status:200 request:0.017s]\n",
            "/usr/local/lib/python3.10/dist-packages/elasticsearch/connection/base.py:208: ElasticsearchWarning: Elasticsearch built-in security features are not enabled. Without authentication, your cluster could be accessible to anyone. See https://www.elastic.co/guide/en/elasticsearch/reference/7.13/security-minimal-setup.html to enable security.\n",
            "  warnings.warn(message, category=ElasticsearchWarning)\n",
            "INFO:elasticsearch:POST http://localhost:9200/geonames/_search [status:200 request:0.026s]\n",
            "2024-02-26 21:00:41,048 elasticsearch INFO     POST http://localhost:9200/geonames/_search [status:200 request:0.026s]\n",
            "<ipython-input-16-07c326d2e514>:14: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
            "  combined_df = combined_df.append(df, ignore_index=True)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'doc_text': 'Schwäbisch Hall is a beautiful town in Baden-Württemberg.',\n",
              " 'event_location_raw': '',\n",
              " 'geolocated_ents': [{'feature_code': 'ADM4',\n",
              "   'feature_class': 'A',\n",
              "   'country_code3': 'DEU',\n",
              "   'lat': 49.11124,\n",
              "   'lon': 9.73947,\n",
              "   'name': 'Schwäbisch Hall',\n",
              "   'admin1_code': '01',\n",
              "   'admin1_name': 'Baden-Wurttemberg',\n",
              "   'admin2_code': '081',\n",
              "   'admin2_name': 'Regierungsbezirk Stuttgart',\n",
              "   'geonameid': '6558007',\n",
              "   'admin1_parent_match': 0,\n",
              "   'country_code_parent_match': 0,\n",
              "   'alt_name_length': 2.0794415416798357,\n",
              "   'min_dist': 0.0,\n",
              "   'max_dist': 0.0,\n",
              "   'avg_dist': 0.0,\n",
              "   'ascii_dist': 0.0,\n",
              "   'adm1_count': 1.0,\n",
              "   'country_count': 1.0,\n",
              "   'score': 0.7057328820228577,\n",
              "   'search_name': 'Schwäbisch Hall',\n",
              "   'start_char': 0,\n",
              "   'end_char': 26,\n",
              "   'city_id': '',\n",
              "   'city_name': ''},\n",
              "  {'feature_code': 'ADM1',\n",
              "   'feature_class': 'A',\n",
              "   'country_code3': 'DEU',\n",
              "   'lat': 48.5,\n",
              "   'lon': 9.0,\n",
              "   'name': 'Baden-Württemberg',\n",
              "   'admin1_code': '01',\n",
              "   'admin1_name': 'Baden-Wurttemberg',\n",
              "   'admin2_code': '',\n",
              "   'admin2_name': '',\n",
              "   'geonameid': '2953481',\n",
              "   'admin1_parent_match': 0,\n",
              "   'country_code_parent_match': 0,\n",
              "   'alt_name_length': 4.465908118654584,\n",
              "   'min_dist': 0.0,\n",
              "   'max_dist': 0.07407407407407407,\n",
              "   'avg_dist': 0.0,\n",
              "   'ascii_dist': 0.0,\n",
              "   'adm1_count': 1.0,\n",
              "   'country_count': 1.0,\n",
              "   'score': 1.0,\n",
              "   'search_name': 'Baden-Württemberg',\n",
              "   'start_char': 39,\n",
              "   'end_char': 62,\n",
              "   'city_id': '',\n",
              "   'city_name': ''}]}"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "combined_df.to_csv(\"satp_2019_geocoded.csv\", index=False)\n",
        "from google.colab import files\n",
        "files.download(\"satp_2019_geocoded.csv\")\n"
      ],
      "metadata": {
        "id": "4vyA1irOqndv",
        "outputId": "1a367678-50f0-4eaa-887e-30d5d78bfde6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "download(\"download_8b0feb03-665a-49d8-b704-b4fd2a4bd7a6\", \"test.csv\", 1057)"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}