Data Privacy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "77b17b78-8c53-468a-9097-82e9c608fd52",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports - these are provided for you. Do not import any other packages.\n",
    "import pandas as pd\n",
    "import requests\n",
    "import bs4\n",
    "from bs4 import BeautifulSoup as bs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "67707b3f-0c51-47f5-bc65-e2e7b64981e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nPart 1: Web Scraping (1.25 points)\\n1a) Web Scrape\\n\\nWe will first retrieve the contents on a page and examine them a bit.\\n\\nMake a variable called wiki, that stores the following URL (as a string): \\n-https://en.wikipedia.org/wiki/List_of_states_and_territories_of_the_United_States_by_population-\\nhttps://en.wikipedia.org/w/index.php?title=List_of_U.S._states_and_territories_by_population&oldid=852357504\\n\\n*** I've changed the url because it was updated and the table is different. \\nIn order to get the same results as the original assingment, I've looked for an older wikipedia version of the info ***\\n\\nNow, to open the URL, use requests.get() and provide wiki as its input. Store this in a variable called page.\\n\\nAfter that, make a variable called soup to parse the HTML using BeautifulSoup. \\nConsider that there will be a method from BeautifulSoup that you’ll need to call on to get the content from the page.\\n\""
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "Part 1: Web Scraping (1.25 points)\n",
    "1a) Web Scrape\n",
    "\n",
    "We will first retrieve the contents on a page and examine them a bit.\n",
    "\n",
    "Make a variable called wiki, that stores the following URL (as a string): \n",
    "-https://en.wikipedia.org/wiki/List_of_states_and_territories_of_the_United_States_by_population-\n",
    "https://en.wikipedia.org/w/index.php?title=List_of_U.S._states_and_territories_by_population&oldid=852357504\n",
    "\n",
    "*** I've changed the url because it was updated and the table is different. \n",
    "In order to get the same results as the original assingment, I've looked for an older wikipedia version of the info ***\n",
    "\n",
    "Now, to open the URL, use requests.get() and provide wiki as its input. Store this in a variable called page.\n",
    "\n",
    "After that, make a variable called soup to parse the HTML using BeautifulSoup. \n",
    "Consider that there will be a method from BeautifulSoup that you’ll need to call on to get the content from the page.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2bffde0d-0629-46bc-9690-a77fe9383df5",
   "metadata": {},
   "outputs": [],
   "source": [
    "wiki = 'https://en.wikipedia.org/w/index.php?title=List_of_U.S._states_and_territories_by_population&oldid=852357504'\n",
    "page = requests.get(wiki)\n",
    "soup = bs(page.content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4f011d02-881b-40a0-b2a0-cb6718d33ff3",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert wiki\n",
    "assert page\n",
    "assert soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5a7c55c0-5150-4ef3-a7ff-5fd66f59fabf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n1b) Checking Scrape Contents\\n\\nExtract the title from the page and save it in a variable called title_page.\\n\\nMake sure you extract it as a string.\\n\\nTo do so, you have to use the soup object created in the above cell. \\nHint: from your soup variable, you can access this with .title.string.\\n\\nMake sure you print out and check the contents of title_page.\\n\\nNote that it should not have any tags (such as <title> included in it).\\n'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "1b) Checking Scrape Contents\n",
    "\n",
    "Extract the title from the page and save it in a variable called title_page.\n",
    "\n",
    "Make sure you extract it as a string.\n",
    "\n",
    "To do so, you have to use the soup object created in the above cell. \n",
    "Hint: from your soup variable, you can access this with .title.string.\n",
    "\n",
    "Make sure you print out and check the contents of title_page.\n",
    "\n",
    "Note that it should not have any tags (such as <title> included in it).\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bdb42984-1378-4540-b8fd-2a2394e49532",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "List of U.S. states and territories by population - Wikipedia\n"
     ]
    }
   ],
   "source": [
    "title_page = soup.title.string\n",
    "print(title_page)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c06b6a86-808d-4cba-b563-94101249d325",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert title_page\n",
    "assert isinstance(title_page, str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d3926e58-4ee6-4308-beb8-7fa6183f457e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n1c) Extracting Tables\\n\\nIn order to extract the data we want, we’ll start with extracting a data table of interest.\\n\\nNote that you can see this table by going to look at the link we scraped.\\n\\nUse the soup object and call a method called find, which will find and extract the first table in the scraped webpage. \\nStore this in the variable right_table.\\n\\nNote: you need to search for the name table, and set the class_ argument as wikitable sortable.\\n'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "1c) Extracting Tables\n",
    "\n",
    "In order to extract the data we want, we’ll start with extracting a data table of interest.\n",
    "\n",
    "Note that you can see this table by going to look at the link we scraped.\n",
    "\n",
    "Use the soup object and call a method called find, which will find and extract the first table in the scraped webpage. \n",
    "Store this in the variable right_table.\n",
    "\n",
    "Note: you need to search for the name table, and set the class_ argument as wikitable sortable.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "309c18cf-f775-4dac-aea3-41612d86f234",
   "metadata": {},
   "outputs": [],
   "source": [
    "right_table = soup.find(class_='wikitable')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "059cb37e-0da2-4524-b178-fe41f9487469",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert right_table\n",
    "assert isinstance(right_table, bs4.element.Tag)\n",
    "assert right_table.name == 'table'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "555d7d79-30f8-4cdd-84ea-0d7d93981101",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "Now, you’ll extract the data from the table into lists.\n",
    "\n",
    "Note: This code is provided for you. Do read through it and try to see how it works.\n",
    "'''\n",
    "\n",
    "# CODE PROVIDED\n",
    "# YOU SHOULD NOT HAVE TO EDIT\n",
    "# BUT YOU WILL WANT TO UNDERSTAND\n",
    "list_a, list_b, list_c = [], [], []\n",
    "\n",
    "for row in right_table.findAll('tr'):\n",
    "    \n",
    "    cells = row.findAll('td')\n",
    "    \n",
    "    # Skips rows that aren't 10 columns long (like the heading)\n",
    "    if len(cells) != 10:\n",
    "        continue\n",
    "\n",
    "    # This catches when the name cells stops having a link\n",
    "    #  and ends, skipping the last (summary rows)\n",
    "    try:\n",
    "        list_a.append(cells[2].find('a').text.strip('\\n'))\n",
    "        list_b.append(cells[3].find(text=True).strip('\\n'))\n",
    "        list_c.append(cells[4].find(text=True).strip('\\n'))\n",
    "    except:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "ad8ee0c8-8efc-4618-855d-f933237eaa86",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n1d) Collecting into a dataframe\\n\\nCreate a dataframe my_df and add the data from the lists above to it.\\n\\nlist_a is the state or territory name. Set the column name as State, and make this the index\\nlist_b is the population estimate. Add it to the dataframe, and set the column name as Population Estimate\\nlist_c is the census population. Add it to the dataframe, and set the column name as Census Population\\nmake sure to check the head of your dataframe to see that everything looks right! ie: my_df.head()\\n'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "1d) Collecting into a dataframe\n",
    "\n",
    "Create a dataframe my_df and add the data from the lists above to it.\n",
    "\n",
    "list_a is the state or territory name. Set the column name as State, and make this the index\n",
    "list_b is the population estimate. Add it to the dataframe, and set the column name as Population Estimate\n",
    "list_c is the census population. Add it to the dataframe, and set the column name as Census Population\n",
    "make sure to check the head of your dataframe to see that everything looks right! ie: my_df.head()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "596868f5-bb47-44c9-ad72-5cd71a4d1082",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Population Estimate</th>\n",
       "      <th>Census Population</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>State</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>California</th>\n",
       "      <td>39,536,653</td>\n",
       "      <td>37,252,895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Texas</th>\n",
       "      <td>28,304,596</td>\n",
       "      <td>25,146,105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Florida</th>\n",
       "      <td>20,984,400</td>\n",
       "      <td>18,804,623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New York</th>\n",
       "      <td>19,849,399</td>\n",
       "      <td>19,378,087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pennsylvania</th>\n",
       "      <td>12,805,537</td>\n",
       "      <td>12,702,887</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Population Estimate Census Population\n",
       "State                                             \n",
       "California            39,536,653        37,252,895\n",
       "Texas                 28,304,596        25,146,105\n",
       "Florida               20,984,400        18,804,623\n",
       "New York              19,849,399        19,378,087\n",
       "Pennsylvania          12,805,537        12,702,887"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = {'State': list_a, 'Population Estimate': list_b, 'Census Population': list_c}\n",
    "my_df = pd.DataFrame(data)\n",
    "my_df = my_df.set_index('State')\n",
    "my_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "33104bde-d492-4018-b06b-216671fddffa",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert isinstance (my_df, pd.DataFrame)\n",
    "assert my_df.index.name == 'State'\n",
    "assert list(my_df.columns) == ['Population Estimate', 'Census Population']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6a7845ad-5e6e-4ca0-92b0-5ee77d069796",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n1e) Using the data\\n\\nWhat is the Population Estimate of California? Save this answer to a variable called ca_pop \\nNotes:\\n\\nExtract this value programmatically from your dataframe (as in, don’t set it explicitly as ca_pop = 123)\\nYou can use .loc to extract a particular value from a dataframe.\\nThe data in your dataframe will be strings - that’s fine, leave them as strings (don’t typecast).\\nStrip any whitespace/newline characters from this string, if necessary. (rstrip() may be helpful)\\n'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "1e) Using the data\n",
    "\n",
    "What is the Population Estimate of California? Save this answer to a variable called ca_pop \n",
    "Notes:\n",
    "\n",
    "Extract this value programmatically from your dataframe (as in, don’t set it explicitly as ca_pop = 123)\n",
    "You can use .loc to extract a particular value from a dataframe.\n",
    "The data in your dataframe will be strings - that’s fine, leave them as strings (don’t typecast).\n",
    "Strip any whitespace/newline characters from this string, if necessary. (rstrip() may be helpful)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0394429b-03f6-469c-b4f9-01182674e63c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39,536,653\n"
     ]
    }
   ],
   "source": [
    "ca_pop = my_df.loc['California']['Population Estimate']\n",
    "print(ca_pop)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "42657fec-78ae-4b48-98c4-4133b3ac4c71",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert ca_pop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3ef74172-cf7c-4e06-bcdb-79566803979d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nPart 2: Identifying Data (3 points)\\n\\nData Files:\\n\\nanon_user_dat.json\\nemployee_info.json\\n\\nYou will first be working with a file called ‘anon_user_dat.json’. \\nThis file contains information about some (fake) Tinder users. \\nWhen creating an account, each Tinder user was asked to provide their first name, last name, \\nwork email (to verify the disclosed workplace), age, gender, phone # and zip code. \\nBefore releasing this data, a data scientist cleaned the data to protect the privacy of Tinder’s users \\nby removing the obvious personal identifiers: phone #, zip code, and IP address. \\nHowever, the data scientist chose to keep each users’ email addresses because when they visually skimmed \\na couple of the email addresses none of them seemed to have any of the users’ actual names in them. \\nThis is where the data scientist made a huge mistake!\\n\\nWe will take advantage of having the work email addresses by finding the employee information of \\ndifferent companies and matching that employee information with the information we have, \\nin order to identify the names of the secret Tinder users!\\n\\n2a) Load in the ‘cleaned’ data\\n\\nLoad the anon_user_dat.json json file into a pandas dataframe. Call it df_personal.\\n'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "Part 2: Identifying Data (3 points)\n",
    "\n",
    "Data Files:\n",
    "\n",
    "anon_user_dat.json\n",
    "employee_info.json\n",
    "\n",
    "You will first be working with a file called ‘anon_user_dat.json’. \n",
    "This file contains information about some (fake) Tinder users. \n",
    "When creating an account, each Tinder user was asked to provide their first name, last name, \n",
    "work email (to verify the disclosed workplace), age, gender, phone # and zip code. \n",
    "Before releasing this data, a data scientist cleaned the data to protect the privacy of Tinder’s users \n",
    "by removing the obvious personal identifiers: phone #, zip code, and IP address. \n",
    "However, the data scientist chose to keep each users’ email addresses because when they visually skimmed \n",
    "a couple of the email addresses none of them seemed to have any of the users’ actual names in them. \n",
    "This is where the data scientist made a huge mistake!\n",
    "\n",
    "We will take advantage of having the work email addresses by finding the employee information of \n",
    "different companies and matching that employee information with the information we have, \n",
    "in order to identify the names of the secret Tinder users!\n",
    "\n",
    "2a) Load in the ‘cleaned’ data\n",
    "\n",
    "Load the anon_user_dat.json json file into a pandas dataframe. Call it df_personal.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d1202888-e902-40cb-b2e8-4c950f6778a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_personal = pd.read_json('anon_user_dat.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "419bb6eb-0011-4845-a4ec-dacc9d12ac12",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert isinstance(df_personal, pd.DataFrame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d61d0f40-e4fd-4b97-9fb2-e962c02582ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n2b) Check the first 10 emails\\n\\nSave the first 10 emails to a Series, and call it sample_emails. You should then print out this Series. ( Use print() )\\n\\nThe purpose of this is to get a sense of how these work emails are structured and how we could possibly \\nextract where each anonymous user seems to work.\\n'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "2b) Check the first 10 emails\n",
    "\n",
    "Save the first 10 emails to a Series, and call it sample_emails. You should then print out this Series. ( Use print() )\n",
    "\n",
    "The purpose of this is to get a sense of how these work emails are structured and how we could possibly \n",
    "extract where each anonymous user seems to work.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b336829a-3996-40da-aecb-f8f0071fb1af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>email</th>\n",
       "      <th>gender</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>60</td>\n",
       "      <td>gshoreson0@seattletimes.com</td>\n",
       "      <td>Male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>47</td>\n",
       "      <td>eweaben1@salon.com</td>\n",
       "      <td>Female</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27</td>\n",
       "      <td>akillerby2@gravatar.com</td>\n",
       "      <td>Male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>46</td>\n",
       "      <td>gsainz3@zdnet.com</td>\n",
       "      <td>Male</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>72</td>\n",
       "      <td>bdanilewicz4@4shared.com</td>\n",
       "      <td>Male</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age                        email  gender\n",
       "0   60  gshoreson0@seattletimes.com    Male\n",
       "1   47           eweaben1@salon.com  Female\n",
       "2   27      akillerby2@gravatar.com    Male\n",
       "3   46            gsainz3@zdnet.com    Male\n",
       "4   72     bdanilewicz4@4shared.com    Male"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_personal.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2eb1a325-692f-49af-b661-780a8ad563df",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    gshoreson0@seattletimes.com\n",
      "1             eweaben1@salon.com\n",
      "2        akillerby2@gravatar.com\n",
      "3              gsainz3@zdnet.com\n",
      "4       bdanilewicz4@4shared.com\n",
      "5      sdeerness5@wikispaces.com\n",
      "6         jstillwell6@ustream.tv\n",
      "7         mpriestland7@opera.com\n",
      "8       nerickssen8@hatena.ne.jp\n",
      "9             hparsell9@xing.com\n",
      "Name: email, dtype: object\n"
     ]
    }
   ],
   "source": [
    "sample_emails = df_personal['email'][0:10]\n",
    "print(sample_emails)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "421a549f-b5bf-4d19-aa0b-de1e21624057",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n2c) Extract the Company Name From the Email\\n\\nCreate a function with the following specifications:\\n\\nFunction Name: extract_company\\nPurpose: to extract the company of the email (i.e., everything after the @ sign but before the first .)\\nParameter(s): email (string)\\nReturns: The extracted part of the email (string)\\nHint: This should take 1 line of code. Look into the find(‘’) method.\\nYou can start with this outline:\\n\\n    def extract_company(email):\\n        return\\n        \\nExample Usage:\\n\\nextract_company(“larhe@uber.com”) should return “uber”\\nextract_company(“ds@cogs.edu”) should return “cogs”\\n'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "2c) Extract the Company Name From the Email\n",
    "\n",
    "Create a function with the following specifications:\n",
    "\n",
    "Function Name: extract_company\n",
    "Purpose: to extract the company of the email (i.e., everything after the @ sign but before the first .)\n",
    "Parameter(s): email (string)\n",
    "Returns: The extracted part of the email (string)\n",
    "Hint: This should take 1 line of code. Look into the find(‘’) method.\n",
    "You can start with this outline:\n",
    "\n",
    "    def extract_company(email):\n",
    "        return\n",
    "        \n",
    "Example Usage:\n",
    "\n",
    "extract_company(“larhe@uber.com”) should return “uber”\n",
    "extract_company(“ds@cogs.edu”) should return “cogs”\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "028d2b4b-1f44-4ce3-8d93-a296eea105c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'seattletimes'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def extract_company(email):\n",
    "    company = []\n",
    "    i = 0\n",
    "    for c in email:\n",
    "        if c == '@':\n",
    "            start = i + 1\n",
    "        elif c == '.':\n",
    "            end = i\n",
    "            break\n",
    "        i += 1\n",
    "    for n in range(start, end):\n",
    "        company.append(email[n])\n",
    "    return ''.join(company)\n",
    "\n",
    "extract_company('gshoreson0@seattletimes.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "e8c8521d-9aac-4576-be9d-4a5306b85f0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert extract_company(\"gshoreson0@seattletimes.com\") == \"seattletimes\"\n",
    "assert extract_company(\"amcgeffen1d@goo.ne.jp\") == 'goo'           "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8a24756a-5bd9-4e46-ad07-e9f4df471531",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nWith a little bit of basic sleuthing (aka googling) and web-scraping (aka selectively reading in html code) \\nit turns out that you’ve been able to collect information about all the present employees/interns of the companies \\nyou are interested in. Specifically, on each company website, you have found the name, gender, and age of its employees. \\nYou have saved that info in employee_info.json and plan to see if, using this new information, \\nyou can match the Tinder accounts to actual names.\\n\\n2d) Load in employee data\\n\\nLoad the json file into a pandas dataframe. Call it df_employee.\\n'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "With a little bit of basic sleuthing (aka googling) and web-scraping (aka selectively reading in html code) \n",
    "it turns out that you’ve been able to collect information about all the present employees/interns of the companies \n",
    "you are interested in. Specifically, on each company website, you have found the name, gender, and age of its employees. \n",
    "You have saved that info in employee_info.json and plan to see if, using this new information, \n",
    "you can match the Tinder accounts to actual names.\n",
    "\n",
    "2d) Load in employee data\n",
    "\n",
    "Load the json file into a pandas dataframe. Call it df_employee.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e83956fe-d239-48c5-9b3d-04a20e34e153",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>company</th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "      <th>gender</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>123-reg</td>\n",
       "      <td>Inglebert</td>\n",
       "      <td>Falconer</td>\n",
       "      <td>Male</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>163</td>\n",
       "      <td>Rafael</td>\n",
       "      <td>Bedenham</td>\n",
       "      <td>Male</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>163</td>\n",
       "      <td>Lemuel</td>\n",
       "      <td>Lind</td>\n",
       "      <td>Male</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>163</td>\n",
       "      <td>Penny</td>\n",
       "      <td>Pennone</td>\n",
       "      <td>Female</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>163</td>\n",
       "      <td>Elva</td>\n",
       "      <td>Crighton</td>\n",
       "      <td>Female</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   company first_name last_name  gender  age\n",
       "0  123-reg  Inglebert  Falconer    Male   42\n",
       "1      163     Rafael  Bedenham    Male   14\n",
       "2      163     Lemuel      Lind    Male   31\n",
       "3      163      Penny   Pennone  Female   45\n",
       "4      163       Elva  Crighton  Female   52"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_employee = pd.read_json('employee_info.json')\n",
    "df_employee.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "bb2d28f0-b73a-4545-8552-10d77db4de8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert isinstance(df_employee, pd.DataFrame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6c32ac7c-81b3-4a43-814a-220a2a52d484",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n2e) Match the employee name with company, age, gender\\n\\nCreate a function with the following specifications:\\n\\nFunction name: employee_matcher\\nPurpose: to match the employee name with the provided company, age, and gender\\nParameter(s): company (string), age (int), gender (string)\\nReturns: The employee first_name and last_name like this: return first_name, last_name\\nNote: If there are multiple employees that fit the same description, first_name and last_name \\nshould return a list of all possible first names and last names i.e., [‘Desmund’, ‘Kelby’], [‘Shepley’, ‘Tichner’]. \\nNote that the names of the individuals that would produce this output are ‘Desmund Shepley’ and ‘Kelby Tichner’.\\nHint: There are many different ways to code this. An inelegant solution is to loop through df_employee \\nand for each data item see if the company, age, and gender match i.e.,\\n\\n    for i in range(0, len(df_employee)):\\n              if (company == df_employee.loc[i,'company']):\\n\\nHowever! The solution above is very inefficient and long, so you should try to look into this: \\nGoogle the df.loc method: It extracts pieces of the dataframe if it fulfills a certain condition. i.e.,\\n\\n    df_employee.loc[df_employee['company'] == company]\\n\\nIf you need to convert your pandas data series into a list, you can do list(result) where result is a pandas “series”\\n\\nYou can start with this outline:\\n\""
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "2e) Match the employee name with company, age, gender\n",
    "\n",
    "Create a function with the following specifications:\n",
    "\n",
    "Function name: employee_matcher\n",
    "Purpose: to match the employee name with the provided company, age, and gender\n",
    "Parameter(s): company (string), age (int), gender (string)\n",
    "Returns: The employee first_name and last_name like this: return first_name, last_name\n",
    "Note: If there are multiple employees that fit the same description, first_name and last_name \n",
    "should return a list of all possible first names and last names i.e., [‘Desmund’, ‘Kelby’], [‘Shepley’, ‘Tichner’]. \n",
    "Note that the names of the individuals that would produce this output are ‘Desmund Shepley’ and ‘Kelby Tichner’.\n",
    "Hint: There are many different ways to code this. An inelegant solution is to loop through df_employee \n",
    "and for each data item see if the company, age, and gender match i.e.,\n",
    "\n",
    "    for i in range(0, len(df_employee)):\n",
    "              if (company == df_employee.loc[i,'company']):\n",
    "\n",
    "However! The solution above is very inefficient and long, so you should try to look into this: \n",
    "Google the df.loc method: It extracts pieces of the dataframe if it fulfills a certain condition. i.e.,\n",
    "\n",
    "    df_employee.loc[df_employee['company'] == company]\n",
    "\n",
    "If you need to convert your pandas data series into a list, you can do list(result) where result is a pandas “series”\n",
    "\n",
    "You can start with this outline:\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "86b208f7-aecd-4dcb-8a7a-c04ee21b6c33",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['Maxwell'], ['Jorio'])"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def employee_matcher(company, age, gender):\n",
    "    match = df_employee.loc[(df_employee['company'] == company) & (df_employee['age'] == age) & (df_employee['gender'] == gender)]\n",
    "    last_name = list(match.last_name)\n",
    "    first_name = list(match.first_name)\n",
    "    return first_name, last_name\n",
    "\n",
    "employee_matcher('google', 41, 'Male')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "6124c8e8-70d9-4654-90d5-287bc1e2e792",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert employee_matcher(\"google\", 41, \"Male\") == (['Maxwell'], ['Jorio'])\n",
    "assert employee_matcher(\"salon\", 47, \"Female\") == (['Elenore'], ['Gravett'])\n",
    "assert employee_matcher(\"webmd\", 28, \"Nonbinary\") == (['Zaccaria'], ['Bartosiak'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "b39516b9-2717-4152-9661-97b87e814bab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n2f) Extract all the private data\\n\\nCreate 2 empty lists called first_names and last_names\\nLoop through all the people we are trying to identify in df_personal\\nCall the extract_company function (i.e., extract_company(df_personal.loc[i, 'email']) )\\nCall the employee_matcher function\\nAppend the results of employee_matcher to the appropriate lists (first_names and last_names)\\n\""
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "2f) Extract all the private data\n",
    "\n",
    "Create 2 empty lists called first_names and last_names\n",
    "Loop through all the people we are trying to identify in df_personal\n",
    "Call the extract_company function (i.e., extract_company(df_personal.loc[i, 'email']) )\n",
    "Call the employee_matcher function\n",
    "Append the results of employee_matcher to the appropriate lists (first_names and last_names)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "ddcea0ba-8f44-4904-86cb-af1446f2223d",
   "metadata": {},
   "outputs": [],
   "source": [
    "first_names = []\n",
    "last_names = []\n",
    "\n",
    "for i in range(len(df_personal)):\n",
    "    company = extract_company(df_personal.loc[i, 'email'])\n",
    "    name = employee_matcher(company, df_personal.loc[i, 'age'], df_personal.loc[i, 'gender'])\n",
    "    first_names.append(name[0])\n",
    "    last_names.append(name[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ed2ddfee-1dd3-409e-a59c-f496eaf100c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert first_names[45:50]== [['Justino'], ['Tadio'], ['Kennith'], ['Cedric'], ['Amargo']]\n",
    "assert last_names[45:50] == [['Corro'], ['Blackford'], ['Milton'], ['Yggo'], ['Grigor']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "b605e31b-0e02-4a15-9508-16d34bc8a03b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n2g) Add the names to the original ‘secure’ dataset!\\n\\nWe have done this last step for you below, all you need to do is run this cell.\\n\\nFor your own personal enjoyment, you should also print out the new df_personal with the identified people.\\n\\ndf_personal['first_name'] = first_names\\ndf_personal['last_name'] = last_names\\n\\nWe have now just discovered the ‘anonymous’ identities of all the registered Tinder users…awkward.\\n\""
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "2g) Add the names to the original ‘secure’ dataset!\n",
    "\n",
    "We have done this last step for you below, all you need to do is run this cell.\n",
    "\n",
    "For your own personal enjoyment, you should also print out the new df_personal with the identified people.\n",
    "\n",
    "df_personal['first_name'] = first_names\n",
    "df_personal['last_name'] = last_names\n",
    "\n",
    "We have now just discovered the ‘anonymous’ identities of all the registered Tinder users…awkward.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "734d75c1-b295-4162-9f22-7dae4fa12340",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>email</th>\n",
       "      <th>gender</th>\n",
       "      <th>first_name</th>\n",
       "      <th>last_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>60</td>\n",
       "      <td>gshoreson0@seattletimes.com</td>\n",
       "      <td>Male</td>\n",
       "      <td>[Gordon]</td>\n",
       "      <td>[DelaField]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>47</td>\n",
       "      <td>eweaben1@salon.com</td>\n",
       "      <td>Female</td>\n",
       "      <td>[Elenore]</td>\n",
       "      <td>[Gravett]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27</td>\n",
       "      <td>akillerby2@gravatar.com</td>\n",
       "      <td>Male</td>\n",
       "      <td>[Abbe]</td>\n",
       "      <td>[Stockdale]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>46</td>\n",
       "      <td>gsainz3@zdnet.com</td>\n",
       "      <td>Male</td>\n",
       "      <td>[Guido]</td>\n",
       "      <td>[Comfort]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>72</td>\n",
       "      <td>bdanilewicz4@4shared.com</td>\n",
       "      <td>Male</td>\n",
       "      <td>[Brody]</td>\n",
       "      <td>[Pinckard]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age                        email  gender first_name    last_name\n",
       "0   60  gshoreson0@seattletimes.com    Male   [Gordon]  [DelaField]\n",
       "1   47           eweaben1@salon.com  Female  [Elenore]    [Gravett]\n",
       "2   27      akillerby2@gravatar.com    Male     [Abbe]  [Stockdale]\n",
       "3   46            gsainz3@zdnet.com    Male    [Guido]    [Comfort]\n",
       "4   72     bdanilewicz4@4shared.com    Male    [Brody]   [Pinckard]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_personal['first_name'] = first_names\n",
    "df_personal['last_name'] = last_names\n",
    "df_personal.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "0235c87c-057e-4a11-a3ea-539f5937bb37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nPart 3: Anonymize Data (3.25 points)\\n\\nYou are hopefully now convinced that with some seemingly harmless data a hacker \\ncan pretty easily discover the identities of certain users. \\nThus, we will now clean the original Tinder data ourselves according to the Safe Harbor Method \\nin order to make sure that it has been properly cleaned…\\n\\n3a) Load in personal data\\n\\nLoad the user_dat.csv file into a pandas dataframe, being sure to read the zip codes (zip) in as a string.\\n\\nStore this in df_users.\\n'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "Part 3: Anonymize Data (3.25 points)\n",
    "\n",
    "You are hopefully now convinced that with some seemingly harmless data a hacker \n",
    "can pretty easily discover the identities of certain users. \n",
    "Thus, we will now clean the original Tinder data ourselves according to the Safe Harbor Method \n",
    "in order to make sure that it has been properly cleaned…\n",
    "\n",
    "3a) Load in personal data\n",
    "\n",
    "Load the user_dat.csv file into a pandas dataframe, being sure to read the zip codes (zip) in as a string.\n",
    "\n",
    "Store this in df_users.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e6cadbfa-4afe-4a8e-a1d9-660b272ebea1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>email</th>\n",
       "      <th>first_name</th>\n",
       "      <th>gender</th>\n",
       "      <th>last_name</th>\n",
       "      <th>ip_address</th>\n",
       "      <th>phone</th>\n",
       "      <th>zip</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>53</td>\n",
       "      <td>co'kenny7a@netscape.com</td>\n",
       "      <td>Cecilla</td>\n",
       "      <td>Female</td>\n",
       "      <td>Seeler</td>\n",
       "      <td>187.127.90.23</td>\n",
       "      <td>(138)263-3068</td>\n",
       "      <td>01012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>52</td>\n",
       "      <td>idrysdell2t@intel.com</td>\n",
       "      <td>Ingmar</td>\n",
       "      <td>Male</td>\n",
       "      <td>Maytum</td>\n",
       "      <td>156.241.220.163</td>\n",
       "      <td>(690)781-2853</td>\n",
       "      <td>01022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>52</td>\n",
       "      <td>bllewellenqr@nhs.uk</td>\n",
       "      <td>Brynne</td>\n",
       "      <td>Female</td>\n",
       "      <td>Edmunds</td>\n",
       "      <td>60.255.124.196</td>\n",
       "      <td>(271)384-3572</td>\n",
       "      <td>01037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>78</td>\n",
       "      <td>myo5x@wikia.com</td>\n",
       "      <td>Marieann</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Dering</td>\n",
       "      <td>37.118.247.19</td>\n",
       "      <td>(754)752-5841</td>\n",
       "      <td>01086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>19</td>\n",
       "      <td>bcashell74@themeforest.net</td>\n",
       "      <td>Burr</td>\n",
       "      <td>Male</td>\n",
       "      <td>Bathersby</td>\n",
       "      <td>201.167.247.226</td>\n",
       "      <td>(771)129-4587</td>\n",
       "      <td>01344</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age                       email first_name  gender  last_name  \\\n",
       "0   53     co'kenny7a@netscape.com    Cecilla  Female     Seeler   \n",
       "1   52       idrysdell2t@intel.com     Ingmar    Male     Maytum   \n",
       "2   52         bllewellenqr@nhs.uk     Brynne  Female    Edmunds   \n",
       "3   78             myo5x@wikia.com   Marieann     NaN     Dering   \n",
       "4   19  bcashell74@themeforest.net       Burr    Male  Bathersby   \n",
       "\n",
       "        ip_address          phone    zip  \n",
       "0    187.127.90.23  (138)263-3068  01012  \n",
       "1  156.241.220.163  (690)781-2853  01022  \n",
       "2   60.255.124.196  (271)384-3572  01037  \n",
       "3    37.118.247.19  (754)752-5841  01086  \n",
       "4  201.167.247.226  (771)129-4587  01344  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_users = pd.read_csv('user_dat.csv', dtype={'zip':str})\n",
    "df_users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "62028582-ed50-4961-956a-ce550c073113",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert isinstance(df_users, pd.DataFrame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "7b65e63c-b90f-4de0-b869-1e62b3d1d124",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n3b) Drop personal attributes\\n\\nRemove any personal information, following the Safe Harbor method. Based on the Safe Harbor method, \\nremove any columns from df_users that contain personal information.\\n\\nNote that details on the Safe Harbor method are covered in the Tutorials.\\n'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "3b) Drop personal attributes\n",
    "\n",
    "Remove any personal information, following the Safe Harbor method. Based on the Safe Harbor method, \n",
    "remove any columns from df_users that contain personal information.\n",
    "\n",
    "Note that details on the Safe Harbor method are covered in the Tutorials.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "cab2e49b-8509-45d5-86a5-dd8e7b7d614b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_users = df_users.drop(columns=['first_name', 'last_name', 'phone', 'email', 'ip_address'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "e71dfe78-ae18-41f5-a5d5-c7cfdf47c32a",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(df_users.columns) == 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "560d094c-2a55-4873-b07f-8187798ae5d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n3c) Drop ages that are above 90\\n\\nSafe Harbor rule C: Drop all the rows which have age greater than 90 from df_users.\\n'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "3c) Drop ages that are above 90\n",
    "\n",
    "Safe Harbor rule C: Drop all the rows which have age greater than 90 from df_users.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "b6583d82-61f5-417c-8153-8101257e5ffd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_users = df_users.loc[df_users['age'] <= 90]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "82a12b6a-77fe-483b-9f65-86f0bbe2e814",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert df_users.shape == (943, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "0ce01a60-a4aa-4144-a0f3-77a0cc90480a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n3d) Load in zip code data\\n\\nLoad the zip_pop.csv file into a (different) pandas dataframe. Call it df_zip.\\n\\nNote that the zip data should be read in as strings, not ints, as would be the default.\\n\\nIn read_csv, use the parameter dtype to specify to read zip as str, and population as int.\\n'"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "3d) Load in zip code data\n",
    "\n",
    "Load the zip_pop.csv file into a (different) pandas dataframe. Call it df_zip.\n",
    "\n",
    "Note that the zip data should be read in as strings, not ints, as would be the default.\n",
    "\n",
    "In read_csv, use the parameter dtype to specify to read zip as str, and population as int.\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "a6c90006-2b8c-4abc-bd6f-0dd8d52ce09f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_zip = pd.read_csv('zip_pop.csv', dtype={'zip':str})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "43d6d2e1-6330-42a2-8b9b-3ba038d3fb80",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert isinstance(df_zip, pd.DataFrame)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "33731de3-59b9-4306-b316-2ba75ea1fe1c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n3e) Sort zipcodes into “Geographic Subdivision”\\n\\nThe Safe Harbor Method applies to “Geographic Subdivisions” as opposed to each zipcode itself.\\n\\nGeographic Subdivision: All areas which share the first 3 digits of a zip code\\n\\nCount the total population for each geographic subdivision, storing the first 3 digits of the zip code and \\nits corresponding population in the dictionary zip_dict. (For example, if there were 20 people whose zip code started with 090, \\nthe key-value pair in zip_dict would be {'090' : 20}.)\\n\\nYou may be tempted to write a gnarly loop to accomplish this. Avoid that temptation. \\nInstead, you’ll want to be savy with a dictionary and groupby from pandas here.\\n\\nTo get you started…\\n\\nIf you wanted to group by whole zip code, you could use something like this:\\n\\ndf_zip.groupby(df_zip['zip'])\\n\\nBut, we don’t want to group by the entire zip code. Instead, we want to extract the first 3 digits of a zip code, \\nand group by that.\\n\\nTo extract the first three digits, you could so something like the following:\\n\\ndf_zip['zip'].str[:3]\\n\\nYou’ll want to combine these two concepts, such that you store this information in a dictionary zip_dict, \\nwhich stores the first three digits of the zip code as the key and the population of that 3-digit zip code as the value.\\n\\n(If you’re stuck and/or to better understand how dictionaries work and how they apply to this concept, \\ncheck the section materials, use google, and go to discussion sections!)\\n\""
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "3e) Sort zipcodes into “Geographic Subdivision”\n",
    "\n",
    "The Safe Harbor Method applies to “Geographic Subdivisions” as opposed to each zipcode itself.\n",
    "\n",
    "Geographic Subdivision: All areas which share the first 3 digits of a zip code\n",
    "\n",
    "Count the total population for each geographic subdivision, storing the first 3 digits of the zip code and \n",
    "its corresponding population in the dictionary zip_dict. (For example, if there were 20 people whose zip code started with 090, \n",
    "the key-value pair in zip_dict would be {'090' : 20}.)\n",
    "\n",
    "You may be tempted to write a gnarly loop to accomplish this. Avoid that temptation. \n",
    "Instead, you’ll want to be savy with a dictionary and groupby from pandas here.\n",
    "\n",
    "To get you started…\n",
    "\n",
    "If you wanted to group by whole zip code, you could use something like this:\n",
    "\n",
    "df_zip.groupby(df_zip['zip'])\n",
    "\n",
    "But, we don’t want to group by the entire zip code. Instead, we want to extract the first 3 digits of a zip code, \n",
    "and group by that.\n",
    "\n",
    "To extract the first three digits, you could so something like the following:\n",
    "\n",
    "df_zip['zip'].str[:3]\n",
    "\n",
    "You’ll want to combine these two concepts, such that you store this information in a dictionary zip_dict, \n",
    "which stores the first three digits of the zip code as the key and the population of that 3-digit zip code as the value.\n",
    "\n",
    "(If you’re stuck and/or to better understand how dictionaries work and how they apply to this concept, \n",
    "check the section materials, use google, and go to discussion sections!)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "c66f4b3b-99f9-4b72-952e-ae8e0904e2d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_zip['zip'] = df_zip['zip'].str.slice(stop=3)\n",
    "zip_dict = df_zip.groupby('zip')['population'].sum().to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "02b07813-5a22-4931-b55e-6443e6c9101c",
   "metadata": {},
   "outputs": [],
   "source": [
    "assert isinstance(zip_dict, dict)\n",
    "assert zip_dict['100'] == 1502501"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "7fd7f9b0-e4ca-4b95-982a-da3821c7cc0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n3f) Masking the Zip Codes\\n\\nIn this part, you should write a for loop, updating the df_users dataframe.\\n\\nGo through each user, and update their zip code, to Safe Harbor specifications:\\n\\nIf the user is from a zip code for the which the “Geographic Subdivision” is less than equal to 20,000, \\nchange the zip code in df_users to ‘0’ (as a string)\\nOtherwise, zip should be only the first 3 numbers of the full zip code\\nDo all this by directly updating the zip column of the df_users DataFrame\\nHints:\\n\\nThis will be several lines of code, looping through the DataFrame, getting each zip code, \\nchecking the geographic subdivision with the population in zip_dict, and setting the zip_code accordingly.\\nBe very aware of your variable types when working with zip codes here.\\n\\n'"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "3f) Masking the Zip Codes\n",
    "\n",
    "In this part, you should write a for loop, updating the df_users dataframe.\n",
    "\n",
    "Go through each user, and update their zip code, to Safe Harbor specifications:\n",
    "\n",
    "If the user is from a zip code for the which the “Geographic Subdivision” is less than equal to 20,000, \n",
    "change the zip code in df_users to ‘0’ (as a string)\n",
    "Otherwise, zip should be only the first 3 numbers of the full zip code\n",
    "Do all this by directly updating the zip column of the df_users DataFrame\n",
    "Hints:\n",
    "\n",
    "This will be several lines of code, looping through the DataFrame, getting each zip code, \n",
    "checking the geographic subdivision with the population in zip_dict, and setting the zip_code accordingly.\n",
    "Be very aware of your variable types when working with zip codes here.\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "0a8521df-be88-42c8-b0e8-7f2d5ae36337",
   "metadata": {},
   "outputs": [],
   "source": [
    "for zipcode in df_users['zip']:\n",
    "    if len(zipcode) != 3:\n",
    "        short_zipcode = zipcode[:len(zipcode) - 2]\n",
    "    if zip_dict[short_zipcode] <= 20000:\n",
    "        df_users.loc[df_users['zip'] == zipcode, 'zip'] = '0'\n",
    "    else:\n",
    "        df_users.loc[df_users['zip'] == zipcode, 'zip'] = short_zipcode\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "18987897-5405-4018-b60a-039d2f5e6a03",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n3g) Save out the properly anonymized data to json file\\n\\nSave out df_users as a json file, called real_anon_user_dat.json\\n'"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "3g) Save out the properly anonymized data to json file\n",
    "\n",
    "Save out df_users as a json file, called real_anon_user_dat.json\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "4aab6a0b-3d05-464d-ae37-1f5dc94628ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_users.to_json('real_anon_user_dat.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8ff11f9-9087-4873-885a-9c2582b53d1f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}