google_search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from helpers import *\n",
    "import difflib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google import google\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "import csv\n",
    "import json\n",
    "from multiprocessing import Pool\n",
    "from sortedcontainers import SortedSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlopen\n",
    "from  urllib.parse import quote_plus\n",
    "import json\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tornado_fetcher import Fetcher\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from google import google\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "import requests\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from importlib import reload\n",
    "import helpers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "csv_filename = \"result.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cities = get_random_cities(3)\n",
    "cities = ['', 'Sydney, Australia', 'Toronto, Canada', 'London, England', 'Bengaluru, India', 'Auckland, New Zealand', 'Florida, United States']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "au_categories = get_data(\"data/google_au_categories.txt\")\n",
    "ca_categories = get_data(\"data/google_ca_categories.txt\")\n",
    "gb_categories = get_data(\"data/google_gb_categories.txt\")\n",
    "in_categories = get_data(\"data/google_in_categories.txt\")\n",
    "nz_categories = get_data(\"data/google_nz_categories.txt\")\n",
    "us_categories = get_data(\"data/google_us_categories.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "partner_categories_1 = get_data(\"data/partner_categories_1.txt\")\n",
    "partner_categories_2 = get_data(\"data/partner_categories_2.txt\")\n",
    "partner_categories_3 = get_data(\"data/partner_categories_3.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "megaset = SortedSet().union(au_categories, ca_categories, gb_categories, in_categories, nz_categories, us_categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "common_categories = megaset.intersection(partner_categories_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# clear_csv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write_result(common_categories, common_categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns=['google_categories','partner_categories']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = pd.DataFrame(columns=columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = pd.DataFrame(list(zip(common_categories, common_categories)), columns = columns)\n",
    "result = result.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "difference = megaset - common_categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = get_close_matches(difference, partner_categories_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# write_result(google_matches, partner_matches)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = result.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "difference -= SortedSet(temp.google_categories.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# temp = get_close_matches(difference, partner_categories_1, 0.94)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# difference -= SortedSet(temp[0].tolist())\n",
    "# result = result.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# flattened_list = []\n",
    "# for matches in partner_matches:\n",
    "#     flattened_list += matches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filename = \"google_mapping.json\"\n",
    "# save_businesses_to_file(difference, filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filename = \"partner_mapping.json\"\n",
    "\n",
    "# from threading import Thread\n",
    "# Thread( target = save_businesses_to_file, args = (partner_categories_1, filename)).start()\n",
    "# save_businesses_to_file(flattened_list, filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get all parent matches (restaurant school manufacturer) c\n",
    "filename = \"google_mapping.json\"\n",
    "google_mapping = load_categories(filename)\n",
    "filename = \"partner_mapping.json\"\n",
    "partner_mapping = load_categories(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = get_matching_categories(google_mapping, partner_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "difference -= SortedSet(temp.google_categories.tolist())\n",
    "result = result.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "translationTable = str.maketrans(\"éàèùâêîôûçã\", \"eaeuaeiouca\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = get_synonym_matches(difference, partner_categories_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = temp[temp.partner_categories.apply(len)>0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "difference -= SortedSet(temp.google_categories.tolist())\n",
    "result = result.append(temp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import matplotlib.pyplot as plt\n",
    "# import numpy as np\n",
    "# %matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from collections import Counter\n",
    "# list1= ' '.join(difference).split()\n",
    "# counts = Counter(list1)\n",
    "# # counts.most_common(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from nltk.corpus import wordnet as wn\n",
    "# import nltk\n",
    "# nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wn.all_synsets('n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from nltk.corpus import wordnet as wn\n",
    "# nouns = {x.name().split('.', 1)[0] for x in wn.all_synsets('n')}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# labels, values = zip(*counts.most_common(20))\n",
    "# labels = list(labels)\n",
    "# values= list(values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i, label in enumerate(labels):\n",
    "#     if re.match('[^a-zA-Z]+', label) is not None or label not in nouns:\n",
    "#         del labels[i]\n",
    "#         del values[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# indexes = np.arange(len(labels))\n",
    "# width = 1\n",
    "# plt.figure(figsize=(24,8))\n",
    "# plt.bar(indexes, values)\n",
    "# plt.xticks(indexes, labels)\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from threading import Thread \n",
    "fetcher = initialize_fetcher()\n",
    "restart_phantomjs()\n",
    "# Thread(target=fetch_categories_from_list, args=(invalid_list,'invalid_categories.json', )).start()\n",
    "fetch_categories_from_list(difference, 'parent_categories.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# results = load_results(\"result.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "biz_categories = load_categories('parent_categories.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1354,
   "metadata": {},
   "outputs": [],
   "source": [
    "rem_categories = load_categories('remaining.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# inv_categories = load_categories('invalid_categories.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# biz_list = [category for category, category_list in biz_categories]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# inv_temp = pd.DataFrame(inv_categories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check = inv_temp[inv_temp[0] == 'andalusian restaurant'].values.tolist()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# invalid_list = [category for category, category_list in inv_categories]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# invalid_cat = list(map(clean, parent_categories))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# invalid_list = [ element[0] for element in list(compress(biz_categories, invalid_cat)) ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parent_category_mapping = pd.DataFrame(list(zip(biz_list, map(get_parent_category, biz_categories))), columns = columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get values with occurunce above 3 in sorted order\n",
    "# loop through each and d chck if xists in result\n",
    "# if does make a tuple of it\n",
    "# use filter and map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# invalid_category_mapping = pd.DataFrame(list(zip(invalid_list, map(get_parent_category, inv_categories))), columns = columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# parent_category_mapping[parent_category_mapping.google_categories.isin(invalid_category_mapping.google_categories)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "parent_category_mapping = parent_category_mapping.append(invalid_category_mapping).drop_duplicates(subset='google_categories', keep='last').sort_values('google_categories')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = parent_category_mapping[parent_category_mapping.google_categories.isin(difference)].drop_duplicates(subset='google_categories', keep='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = temp.query('google_categories != partner_categories')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = temp[~temp.partner_categories.isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_result = result[result.google_categories.isin(temp.partner_categories)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df = pd.merge(temp, temp_result,  how = 'right', left_on = 'partner_categories', right_on = 'google_categories')[['google_categories_x', 'partner_categories_y']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.columns = columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = result.append(final_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "difference -= SortedSet(final_df.google_categories.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1446,
   "metadata": {},
   "outputs": [],
   "source": [
    "category_parent_pd = pd.DataFrame(rem_categories, columns = columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1448,
   "metadata": {},
   "outputs": [],
   "source": [
    "most_occuring = list(map(get_most_occuring_categories, category_parent_pd.values.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1449,
   "metadata": {},
   "outputs": [],
   "source": [
    "def exists_in_result(categories):\n",
    "    if not categories:\n",
    "        return np.nan\n",
    "    for category in categories:\n",
    "        if category in result.google_categories.tolist():\n",
    "            return category\n",
    "    return np.nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1450,
   "metadata": {},
   "outputs": [],
   "source": [
    "parent_categories = pd.DataFrame(list(zip( category_parent_pd.google_categories.tolist(), map(exists_in_result, most_occuring))), columns= ['google_categories','parent_categories'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list(zip( category_parent_pd.google_categories.tolist(), most_occuring))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1411,
   "metadata": {},
   "outputs": [],
   "source": [
    "parent_categories = parent_categories[parent_categories.parent_categories.notnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1417,
   "metadata": {},
   "outputs": [],
   "source": [
    "# temp.merge(parent_categories)\n",
    "temp = pd.merge(category_parent_pd, parent_categories,  how = 'inner', on = 'google_categories')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1419,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_result = result[result.google_categories.isin(temp.parent_categories)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1421,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df = pd.merge(temp, temp_result,  how = 'right', left_on = 'parent_categories', right_on = 'google_categories')[['google_categories_x', 'partner_categories_y']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1422,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.columns = columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1424,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = result.append(final_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1425,
   "metadata": {},
   "outputs": [],
   "source": [
    "difference -= SortedSet(final_df.google_categories.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i, category in enumerate(no_second):\n",
    "#     try:\n",
    "#         biz_names = google_mapping[category]\n",
    "#     except:\n",
    "#         continue\n",
    "        \n",
    "#     print(\"{} {}\".format(i, category))\n",
    "        \n",
    "#     cat_results = []\n",
    "    \n",
    "#     if len(biz_names) > 10:\n",
    "#         sample = random.sample(biz_names, 10)\n",
    "#     else:\n",
    "#         sample = biz_names\n",
    "    \n",
    "#     for biz_name in sample:\n",
    "#         biz_results = get_facebook_category(biz_name)\n",
    "#         print(biz_name, biz_results)\n",
    "#         cat_results += biz_results\n",
    "        \n",
    "#     category_mapping.append({category: cat_results})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('facebook_mapping.json', 'w') as file:\n",
    "#     file.write(json.dumps(category_mapping))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1269,
   "metadata": {},
   "outputs": [],
   "source": [
    "facebook_mapping = load_categories('facebook_mapping.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1284,
   "metadata": {},
   "outputs": [],
   "source": [
    "fb_categories = get_facebook_df(facebook_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1285,
   "metadata": {},
   "outputs": [],
   "source": [
    "fb_categories = fb_categories[fb_categories.google_categories.isin(difference)].drop_duplicates(subset='google_categories', keep='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1287,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = get_close_matches(fb_categories.facebook_categories.drop_duplicates().tolist(), result.google_categories.tolist(), 0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1288,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = temp.apply(lambda x: [x[0], x[1][0]], axis=1, result_type='expand')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1289,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp.columns = ['facebook_categories', 'result_categories']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1291,
   "metadata": {},
   "outputs": [],
   "source": [
    "last = pd.merge(fb_categories, temp, how='inner', on='facebook_categories').drop_duplicates(subset='google_categories', keep='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1293,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_result = result[result.google_categories.isin(last.result_categories)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1255,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df = pd.merge(last, temp_result,  how = 'right', left_on = 'result_categories', right_on = 'google_categories')[['google_categories_x', 'partner_categories']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1256,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.columns = columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1297,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>google_categories</th>\n",
       "      <th>partner_categories</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [google_categories, partner_categories]\n",
       "Index: []"
      ]
     },
     "execution_count": 1297,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df = final_df[~final_df.google_categories.isin(result.google_categories)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1276,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = result.append(final_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1277,
   "metadata": {},
   "outputs": [],
   "source": [
    "difference -= SortedSet(final_df.google_categories.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1458,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "65"
      ]
     },
     "execution_count": 1458,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(difference)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fetch_business_categories('tiffin service provider')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1335,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 1335,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fetch_businesses('metalware producer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1340,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('difference.json', 'w') as file:\n",
    "    file.write(json.dumps(list(difference)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"last.json\"\n",
    "save_businesses_to_file(difference, filename)\n",
    "# Thread( target = save_businesses_to_file, args = (difference, filename)).start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!curl --upload-file no_second.json https://transfer.sh/no_second.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1469,
   "metadata": {},
   "outputs": [],
   "source": [
    "reload(helpers)\n",
    "from helpers import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1461,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = result.drop_duplicates(subset='google_categories', keep='last')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1462,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5013, 2)"
      ]
     },
     "execution_count": 1462,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1459,
   "metadata": {},
   "outputs": [],
   "source": [
    "result.to_csv('pandas_result.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1348,
   "metadata": {},
   "outputs": [],
   "source": [
    "invalid_list = result[result.partner_categories.str.match(\"[\\W]+\", na =False)].google_categories.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1383,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = pd.read_csv('pandas_result.csv')[['google_categories','partner_categories']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}