Geoparsing.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Geoparsing Workbook"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Named Entity Recognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import spacy\n",
    "from spacy import displacy\n",
    "from collections import Counter\n",
    "import en_core_web_lg\n",
    "nlp = en_core_web_lg.load()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Geoparser requires ElastiSearch container running"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mordecai import Geoparser\n",
    "geo = Geoparser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#this takes like 8 hours to run\n",
    "def geoparsehelper(tweet):    \n",
    "    try:\n",
    "        result = geo.geoparse(tweet)\n",
    "    except:\n",
    "        result = []\n",
    "    return result\n",
    "\n",
    "data[\"place\"] = data.apply(lambda x: geoparsehelper(x[\"tweet_text_processed\"]),axis=1)\n",
    "# drop columns that only have NaNs\n",
    "data = data.dropna(axis=1, how='all')\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save output of geoparse operation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv(\"redforedwithlocation.csv\",dtype)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load output "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "data = pd.read_csv('redforedwithlocation.csv')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert PLACE to LAT and LON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filterhelp(x):\n",
    "    try: \n",
    "        return len(x) > 2\n",
    "    except: \n",
    "        return False\n",
    "\n",
    "\n",
    "filterdf = data[data['place'].apply(lambda x: filterhelp(x))]\n",
    "\n",
    "def lathelper(geo):\n",
    "    try:\n",
    "        return float(geo[geo.find('lat')+7:geo.find(',',geo.find('lat'))-1])\n",
    "    except:\n",
    "        pass\n",
    "        \n",
    "def lonhelper(geo):\n",
    "    try:\n",
    "        return float(geo[geo.find('lon')+7:geo.find(',',geo.find('lon'))-1])\n",
    "    except:\n",
    "        pass\n",
    "\n",
    "filterdf[\"lat\"] = filterdf[\"place\"].apply(lambda x: lathelper(x))\n",
    "filterdf[\"lon\"] = filterdf[\"place\"].apply(lambda x: lonhelper(x))\n",
    "filterdf = filterdf[filterdf[\"lat\"].notnull()]\n",
    "#filterdf removes all tweets that don't have a location reference according to the geoparsing process"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Static map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.animation as animation\n",
    "import geopandas\n",
    "import datetime\n",
    "\n",
    "geodf = geopandas.GeoDataFrame(filterdf,crs=\"EPSG:4326\",geometry=geopandas.points_from_xy(filterdf['lon'], filterdf['lat']))\n",
    "geodf = geodf.to_crs(\"EPSG:5070\")\n",
    "# 5070 for Albers US focus\n",
    "# 3395 for Mercator Global focus\n",
    "#globe = geopandas.read_file('shapefiles/world_countries_2020.shp')\n",
    "states = geopandas.read_file('shapefiles/cb_2018_us_state_20m.shp')\n",
    "states = states.to_crs(\"EPSG:5070\")\n",
    "geodf = geopandas.clip(geodf,states)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(50,20))\n",
    "states.plot(ax=ax, color='white', edgecolor='black')\n",
    "geodf.plot(ax=ax, marker='o', color='red', markersize=5)\n",
    "plt.show();"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Animated GIF of map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.animation as animation\n",
    "import geopandas\n",
    "import datetime\n",
    "\n",
    "geodf = geopandas.GeoDataFrame(filterdf,crs=\"EPSG:4326\",geometry=geopandas.points_from_xy(filterdf['lon'], filterdf['lat']))\n",
    "geodf = geodf.to_crs(\"EPSG:5070\")\n",
    "geodf.date = pd.to_datetime(geodf.date, format='%m/%d/%y')\n",
    "\n",
    "states = geopandas.read_file('shapefiles/cb_2018_us_state_20m.shp')\n",
    "states = states.to_crs(\"EPSG:5070\")\n",
    "states = states[(states[\"NAME\"] != \"Alaska\") & (states[\"NAME\"] != \"Puerto Rico\") & (states[\"NAME\"] != \"Hawaii\")]\n",
    "geodf = geopandas.clip(geodf,states)\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(50,20))\n",
    "\n",
    "\n",
    "renderdate = pd.to_datetime('02/01/18',format='%m/%d/%y')\n",
    "startdate = pd.to_datetime('02/01/18',format='%m/%d/%y')\n",
    "\n",
    "# initialization function \n",
    "def init(): \n",
    "    states.plot(ax=ax, color='white', edgecolor='black')\n",
    "    \n",
    "# animation function \n",
    "def animate(i): \n",
    "    global renderdate\n",
    "    renderdate = renderdate + datetime.timedelta(days=7)\n",
    "    mask = (geodf['date'] > startdate) & (geodf['date'] <= renderdate)\n",
    "    rendergeodf = geodf.loc[mask]\n",
    "    rendergeodf.plot(ax=ax, marker='o', color='red', markersize=5)\n",
    "    plt.title(renderdate)\n",
    "\n",
    "anim = animation.FuncAnimation(fig, animate, init_func=init, interval=250, frames=120) \n",
    "anim.save('strikes.gif', writer='ffmpeg')\n",
    "plt.show();"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bokeh map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bokeh.io import output_file, show\n",
    "from bokeh.plotting import figure\n",
    "from bokeh.models import ColumnDataSource, HoverTool, LinearColorMapper, ColorBar\n",
    "from bokeh.palettes import Viridis256\n",
    "from bokeh.tile_providers import get_provider, Vendors\n",
    "from pyproj import Transformer\n",
    "from bokeh.plotting import figure, output_file, save\n",
    "\n",
    "# Example dataframe with latitude and longitude coordinates\n",
    "# Convert lat/lon to Web Mercator projection\n",
    "transformer = Transformer.from_crs(\"EPSG:4326\", \"EPSG:3857\")\n",
    "x, y = transformer.transform(filterdf['lat'].tolist(), filterdf['lon'].tolist())\n",
    "\n",
    "# Create ColumnDataSource\n",
    "df = pd.DataFrame()\n",
    "df['label'] = filterdf['tweet_text_processed'].astype(str)\n",
    "source = ColumnDataSource(data = dict(x=x, y=y, label=df['label'].tolist()))\n",
    "map_provider = get_provider(Vendors.CARTODBPOSITRON)\n",
    "\n",
    "# Set up Bokeh plot\n",
    "p = figure(\n",
    "    x_range = (-13884029,-7453304),\n",
    "    y_range = (2698291,6455972),\n",
    "    x_axis_type=\"mercator\",\n",
    "    y_axis_type=\"mercator\",\n",
    "    sizing_mode='stretch_both',\n",
    "    background_fill_color='whitesmoke',\n",
    "    border_fill_color='whitesmoke'\n",
    ")\n",
    "p.add_tile(map_provider)\n",
    "\n",
    "# Remove axis and tickers\n",
    "p.xaxis.visible = False\n",
    "p.yaxis.visible = False\n",
    "p.xgrid.grid_line_color = None\n",
    "p.ygrid.grid_line_color = None\n",
    "\n",
    "# Add circles to plot\n",
    "\n",
    "p.circle(x='x', y='y', size=5, fill_color='red', fill_alpha=0.1, line_color=\"black\", source=source,legend_label='Tweets')\n",
    "hover = HoverTool(tooltips=[('Tweet', '@label')])\n",
    "p.add_tools(hover)\n",
    "# Add legend\n",
    "p.legend.location = 'top_right'\n",
    "\n",
    "# Save plot as HTML file\n",
    "output_file('index.html')\n",
    "save(p)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ds",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "64a3776662997a24a2ed087fc314d1fb0ced86b8ae78a59009a03d7b36ffebb7"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}