04_clean-and-geocode-locations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Clean and geocode locations\n",
    "Clean and geocode (convert from strings to structured results, including geometries such as point coordinates) the locations found in the previous step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "\n",
    "# own files\n",
    "from pysci import docutils as du\n",
    "from pysci import geoparse as gp\n",
    "from pysci import geocode as gc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preliminaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>content_locations</th>\n",
       "      <th>use_xml</th>\n",
       "      <th>location_sentences</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Liu_et_al-2015-Insect_Conservation_and_Diversity</td>\n",
       "      <td>Xitiange</td>\n",
       "      <td>True</td>\n",
       "      <td>This study was conducted at Xitiange village i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Liu_et_al-2015-Insect_Conservation_and_Diversity</td>\n",
       "      <td>Northeastern Miyun County (</td>\n",
       "      <td>True</td>\n",
       "      <td>This study was conducted at Xitiange village i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Liu_et_al-2015-Insect_Conservation_and_Diversity</td>\n",
       "      <td>Beijing city</td>\n",
       "      <td>True</td>\n",
       "      <td>The area is situated about 70 km north of Beij...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Liu_et_al-2015-Insect_Conservation_and_Diversity</td>\n",
       "      <td>Xitiange</td>\n",
       "      <td>True</td>\n",
       "      <td>This study was conducted at Xitiange village i...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Liu_et_al-2015-Insect_Conservation_and_Diversity</td>\n",
       "      <td>Woodland</td>\n",
       "      <td>True</td>\n",
       "      <td>Woodland was also planted with Populus spp., b...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           filename  \\\n",
       "0  Liu_et_al-2015-Insect_Conservation_and_Diversity   \n",
       "1  Liu_et_al-2015-Insect_Conservation_and_Diversity   \n",
       "2  Liu_et_al-2015-Insect_Conservation_and_Diversity   \n",
       "3  Liu_et_al-2015-Insect_Conservation_and_Diversity   \n",
       "4  Liu_et_al-2015-Insect_Conservation_and_Diversity   \n",
       "\n",
       "             content_locations  use_xml  \\\n",
       "0                     Xitiange     True   \n",
       "1  Northeastern Miyun County (     True   \n",
       "2                 Beijing city     True   \n",
       "3                     Xitiange     True   \n",
       "4                     Woodland     True   \n",
       "\n",
       "                                  location_sentences  \n",
       "0  This study was conducted at Xitiange village i...  \n",
       "1  This study was conducted at Xitiange village i...  \n",
       "2  The area is situated about 70 km north of Beij...  \n",
       "3  This study was conducted at Xitiange village i...  \n",
       "4  Woodland was also planted with Populus spp., b...  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path_to_content_locations = os.path.join('results', 'locations.tsv')\n",
    "df_locations = pd.read_csv(path_to_content_locations, sep='\\t', quotechar='\"', encoding='utf-8')\n",
    "df_locations.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create googlemaps.Client object: you will need an API key for this!\n",
    "gmaps = gc.create_google_geocoder(api_key='YOUR-API-KEY-HERE')\n",
    "# cache so we don't re-query identical strings\n",
    "local_cache_google = {}\n",
    "# or load existing cache\n",
    "#local_cache_google = du.load_data('local_cache_google.pkl')\n",
    "#print(\"The cache has values for %s string keys.\" %len(local_cache_google))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Geocode locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done.\n"
     ]
    }
   ],
   "source": [
    "loc_clean = []\n",
    "loc_lats = []\n",
    "loc_lons = []\n",
    "loc_strings = []\n",
    "loc_types = []\n",
    "\n",
    "for content_location in df_locations['content_locations'].tolist():\n",
    "    if content_location == gp.NO_METHODS_STRING or content_location == gp.NO_LOCATIONS_STRING:\n",
    "        # don't geocode these but store placeholder strings\n",
    "        loc_clean.append(content_location)\n",
    "        loc_lats.append(content_location)\n",
    "        loc_lons.append(content_location)\n",
    "        loc_strings.append(content_location)\n",
    "        loc_types.append(content_location)\n",
    "        continue\n",
    "    clean_text = gc.clean_for_geocode(content_location)\n",
    "    loc_clean.append(clean_text)\n",
    "    #print(\"\\tgeocoding text: %s\" %clean_text)\n",
    "    top = gc.geocode_with_cache_google(clean_text, gmaps, local_cache_google)\n",
    "    if top:\n",
    "        # get point\n",
    "        geometry = top['geometry']\n",
    "        pt = geometry['location']\n",
    "        # store stuff\n",
    "        loc_lats.append(pt['lat'])\n",
    "        loc_lons.append(pt['lng'])\n",
    "        loc_strings.append(top['formatted_address'])\n",
    "        loc_types.append(geometry['location_type'])\n",
    "    else:\n",
    "        #print(\"\\tWe had no top result\")\n",
    "        loc_lats.append(gc.NO_RESULT_STRING)\n",
    "        loc_lons.append(gc.NO_RESULT_STRING)\n",
    "        loc_strings.append(gc.NO_RESULT_STRING)\n",
    "        loc_types.append(gc.NO_RESULT_STRING)\n",
    "\n",
    "print(\"Done.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The cache now has values for 5 string keys.\n",
      "pickled data at local_cache_google.pkl\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# repickle the cache at the end...\n",
    "print(\"The cache now has values for %s string keys.\" %len(local_cache_google))\n",
    "du.pickle_data(local_cache_google, 'local_cache_google.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tidy and export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>content_locations</th>\n",
       "      <th>use_xml</th>\n",
       "      <th>location_sentences</th>\n",
       "      <th>clean_content_loc</th>\n",
       "      <th>geocode_lat</th>\n",
       "      <th>geocode_lon</th>\n",
       "      <th>geocode_str</th>\n",
       "      <th>geocode_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Liu_et_al-2015-Insect_Conservation_and_Diversity</td>\n",
       "      <td>Xitiange</td>\n",
       "      <td>True</td>\n",
       "      <td>This study was conducted at Xitiange village i...</td>\n",
       "      <td>Xitiange</td>\n",
       "      <td>no-geocode-result</td>\n",
       "      <td>no-geocode-result</td>\n",
       "      <td>no-geocode-result</td>\n",
       "      <td>no-geocode-result</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Liu_et_al-2015-Insect_Conservation_and_Diversity</td>\n",
       "      <td>Northeastern Miyun County (</td>\n",
       "      <td>True</td>\n",
       "      <td>This study was conducted at Xitiange village i...</td>\n",
       "      <td>Northeastern Miyun County</td>\n",
       "      <td>40.3769</td>\n",
       "      <td>116.843</td>\n",
       "      <td>Miyun, Beijing, China</td>\n",
       "      <td>APPROXIMATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Liu_et_al-2015-Insect_Conservation_and_Diversity</td>\n",
       "      <td>Beijing city</td>\n",
       "      <td>True</td>\n",
       "      <td>The area is situated about 70 km north of Beij...</td>\n",
       "      <td>Beijing city</td>\n",
       "      <td>39.9042</td>\n",
       "      <td>116.407</td>\n",
       "      <td>Beijing, China</td>\n",
       "      <td>APPROXIMATE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Liu_et_al-2015-Insect_Conservation_and_Diversity</td>\n",
       "      <td>Xitiange</td>\n",
       "      <td>True</td>\n",
       "      <td>This study was conducted at Xitiange village i...</td>\n",
       "      <td>Xitiange</td>\n",
       "      <td>no-geocode-result</td>\n",
       "      <td>no-geocode-result</td>\n",
       "      <td>no-geocode-result</td>\n",
       "      <td>no-geocode-result</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Liu_et_al-2015-Insect_Conservation_and_Diversity</td>\n",
       "      <td>Woodland</td>\n",
       "      <td>True</td>\n",
       "      <td>Woodland was also planted with Populus spp., b...</td>\n",
       "      <td>Woodland</td>\n",
       "      <td>45.9046</td>\n",
       "      <td>-122.744</td>\n",
       "      <td>Woodland, WA 98674, USA</td>\n",
       "      <td>APPROXIMATE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           filename  \\\n",
       "0  Liu_et_al-2015-Insect_Conservation_and_Diversity   \n",
       "1  Liu_et_al-2015-Insect_Conservation_and_Diversity   \n",
       "2  Liu_et_al-2015-Insect_Conservation_and_Diversity   \n",
       "3  Liu_et_al-2015-Insect_Conservation_and_Diversity   \n",
       "4  Liu_et_al-2015-Insect_Conservation_and_Diversity   \n",
       "\n",
       "             content_locations  use_xml  \\\n",
       "0                     Xitiange     True   \n",
       "1  Northeastern Miyun County (     True   \n",
       "2                 Beijing city     True   \n",
       "3                     Xitiange     True   \n",
       "4                     Woodland     True   \n",
       "\n",
       "                                  location_sentences  \\\n",
       "0  This study was conducted at Xitiange village i...   \n",
       "1  This study was conducted at Xitiange village i...   \n",
       "2  The area is situated about 70 km north of Beij...   \n",
       "3  This study was conducted at Xitiange village i...   \n",
       "4  Woodland was also planted with Populus spp., b...   \n",
       "\n",
       "           clean_content_loc        geocode_lat        geocode_lon  \\\n",
       "0                   Xitiange  no-geocode-result  no-geocode-result   \n",
       "1  Northeastern Miyun County            40.3769            116.843   \n",
       "2               Beijing city            39.9042            116.407   \n",
       "3                   Xitiange  no-geocode-result  no-geocode-result   \n",
       "4                   Woodland            45.9046           -122.744   \n",
       "\n",
       "               geocode_str       geocode_type  \n",
       "0        no-geocode-result  no-geocode-result  \n",
       "1    Miyun, Beijing, China        APPROXIMATE  \n",
       "2           Beijing, China        APPROXIMATE  \n",
       "3        no-geocode-result  no-geocode-result  \n",
       "4  Woodland, WA 98674, USA        APPROXIMATE  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_locations['clean_content_loc'] = loc_clean\n",
    "df_locations['geocode_lat'] = loc_lats\n",
    "df_locations['geocode_lon'] = loc_lons\n",
    "df_locations['geocode_str'] = loc_strings\n",
    "df_locations['geocode_type'] = loc_types\n",
    "df_locations.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_in_order = ['filename', 'content_locations', 'clean_content_loc', 'geocode_str', 'geocode_type', \n",
    "                 'geocode_lat', 'geocode_lon', 'use_xml', 'location_sentences']\n",
    "df_locations_clean = df_locations.loc[:,cols_in_order]\n",
    "# export to file\n",
    "df_locations_clean.to_csv(os.path.join('results', 'locations_geocoded.tsv'), sep='\\t', index=False, quotechar='\"', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}