14_rules_linear-combination.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Rule-based gazetteer matching: linear-combination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from leven import levenshtein\n",
    "    \n",
    "# own file\n",
    "from gazmatch import matchutils as mu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dir = 'data'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "full: 468 pairs\n"
     ]
    }
   ],
   "source": [
    "# load ground truth data (pickled)\n",
    "ground_truth_flat = mu.load_data(os.path.join(data_dir, 'ground_truth_flat.pkl'))\n",
    "print(\"full: %s pairs\" %len(ground_truth_flat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(313562, 21)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load SwissNames3D augmented with lat, lons\n",
    "path_swissnames_csv = os.path.join(data_dir, 'swissnames_new_uuids.csv')\n",
    "swissnames = pd.read_csv(path_swissnames_csv, sep='\\t', encoding='utf-8', low_memory=False)\n",
    "swissnames.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# source data (Geonames records)\n",
    "path_to_source_data = os.path.join(data_dir, 'source_data_augmented.csv')\n",
    "df_source = pd.read_csv(path_to_source_data, sep='\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rule-based matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for use on a row from swissnames, and a geonames name ('geoname')\n",
    "def get_edit_distance(row, geoname):\n",
    "    sn_name = row['NAME']\n",
    "    leven_dist = levenshtein(geoname, sn_name)\n",
    "    return leven_dist\n",
    "\n",
    "def similarity_points(row):\n",
    "    weight1 = 0.5\n",
    "    weight2 = 0.5\n",
    "    sim_score = weight1*(row['leven_dist']) + weight2*(row['distances'])\n",
    "    return sim_score\n",
    "\n",
    "def similarity_lines_pols(row):\n",
    "    weight1 = 0.7\n",
    "    weight2 = 0.3\n",
    "    sim_score = weight1*(row['leven_dist']) + weight2*row['distances']\n",
    "    return sim_score\n",
    "\n",
    "# settings and prep\n",
    "similarity_threshold = 2.0\n",
    "\n",
    "# soft feature type alignments\n",
    "soft_type_align = {}\n",
    "soft_type_align['LK'] = ['Seeteil', 'See']\n",
    "soft_type_align['GLCR'] = ['Gletscher', 'Alpiner Gipfel']\n",
    "soft_type_align['STM'] = ['Fliessgewaesser']\n",
    "soft_type_align['PK'] = ['Hauptgipfel', 'Gipfel', 'Huegel', 'Haupthuegel', 'Alpiner Gipfel', 'Grat', 'Huegelzug', 'Felskopf']\n",
    "soft_type_align['PASS'] = ['Pass', 'Graben']\n",
    "soft_type_align['HLL'] = ['Hauptgipfel', 'Gipfel', 'Huegel', 'Haupthuegel', 'Alpiner Gipfel', 'Grat', 'Huegelzug', 'Felskopf']\n",
    "soft_type_align['MT'] = ['Hauptgipfel', 'Gipfel', 'Huegel', 'Haupthuegel', 'Alpiner Gipfel', 'Grat', 'Huegelzug', 'Felskopf']\n",
    "soft_type_align['VAL'] = ['Tal', 'Haupttal', 'Graben']\n",
    "len(soft_type_align)\n",
    "\n",
    "# swissnames name groups\n",
    "sn_name_groups = swissnames.groupby(['NAME'])\n",
    "\n",
    "# geonames type groups\n",
    "type_groups = df_source.groupby(['feature code'])\n",
    "\n",
    "points = ['PK', 'HLL', 'MT', 'GLCR', 'PASS']\n",
    "lines_pols = ['STM', 'LK', 'VAL']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prec\trecall\tf1\n",
      "0.871\t0.833\t0.852\n"
     ]
    }
   ],
   "source": [
    "matches_overall = []\n",
    "neg_overall = []\n",
    "results_type = {}\n",
    "for ftype, group in type_groups:\n",
    "    matches_type = []\n",
    "    neg_type = []\n",
    "    gn_ids_type = []\n",
    "    for index, gn_record in group.iterrows():\n",
    "        name = gn_record['name']\n",
    "        alternatenames = str(gn_record['alternatenames'])\n",
    "        gn_type = gn_record['feature code']\n",
    "        gn_point = (gn_record['latitude'], gn_record['longitude'])\n",
    "        gn_id = gn_record['geonameid']\n",
    "        gn_ids_type.append(gn_id)\n",
    "        # get any exact name matches as candidates, including altnames and the comma-flip trick\n",
    "        df_exact_all = mu.find_all_exact_name_matches(name, alternatenames, sn_name_groups, swissnames)\n",
    "        df_exact = df_exact_all.copy()\n",
    "        df_exact['leven_dist'] = 0\n",
    "        # consider only features of compatible types for the rest: could also replace with a very coarse spatial filter\n",
    "        df_candidates = swissnames[swissnames['OBJEKTART'].isin(soft_type_align[gn_type])].copy()\n",
    "        # calculate levenshtein distance for just feature-type candidates\n",
    "        df_candidates['leven_dist'] = df_candidates.apply(lambda r: get_edit_distance(r, name), axis=1)\n",
    "        # add the exact matches and calculate geo-distances\n",
    "        df_candidates = df_candidates.append(df_exact)  # don't drop duplicates because we lose info\n",
    "        df_candidates['distances'] = df_candidates.apply(lambda r: mu.calculate_distance_local(r, gn_point), axis=1)\n",
    "        # combine text distance with geodistance for a total score (two ways to combine depending on feature type)\n",
    "        if gn_type in points:\n",
    "            df_candidates['similarity'] = df_candidates.apply(similarity_points, axis=1)\n",
    "        elif gn_type in lines_pols:\n",
    "            df_candidates['similarity'] = df_candidates.apply(similarity_lines_pols, axis=1)\n",
    "        # keep false positives low (i.e. keep precision high)\n",
    "        df_subset = df_candidates[df_candidates['similarity'] < similarity_threshold]\n",
    "        df_subset = df_subset.drop_duplicates()\n",
    "        if df_subset.shape[0] > 0:\n",
    "            for sn_id in df_subset['UUID_old'].tolist():\n",
    "                matches_type.append((gn_id, sn_id))\n",
    "                matches_overall.append((gn_id, sn_id))\n",
    "        else:\n",
    "            # no features were below threshold\n",
    "            neg_type.append((gn_id, mu.NO_MATCH_STRING))\n",
    "            neg_overall.append((gn_id, mu.NO_MATCH_STRING))\n",
    "    \n",
    "    # filter ground truth to ids of current feature type only\n",
    "    gn_ids_type_set = set(gn_ids_type)\n",
    "    ground_truth_type = [item for item in ground_truth_flat if item[0] in gn_ids_type_set]\n",
    "    results_type[ftype] = mu.evaluate_results_list(matches_type, neg_type, ground_truth_type, verbose=False)\n",
    "\n",
    "print(\"prec\\trecall\\tf1\")\n",
    "p, r, f1 = mu.evaluate_results_list(matches_overall, neg_overall, ground_truth_flat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Results by type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "type\tprec\trecall\tf1\n",
      "GLCR\t0.966\t0.966\t0.966\n",
      "HLL\t0.897\t0.981\t0.937\n",
      "LK\t0.855\t1.000\t0.922\n",
      "MT\t0.759\t0.978\t0.854\n",
      "PASS\t0.962\t0.962\t0.962\n",
      "PK\t0.902\t0.958\t0.929\n",
      "STM\t0.605\t0.271\t0.374\n",
      "VAL\t0.905\t0.844\t0.874\n"
     ]
    }
   ],
   "source": [
    "print(\"type\\tprec\\trecall\\tf1\")\n",
    "for k,v in results_type.items():\n",
    "    print(\"%s\\t%.3f\\t%.3f\\t%.3f\" %(k, v[0], v[1], v[2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}