8 - Introduction to AWS Boto in Python.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "1 - Putting Files in the Cloud!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Your first boto3 client\n",
    "\n",
    "# Generate the boto3 client for interacting with S3\n",
    "s3 = boto3.client('s3', region_name='us-east-1', \n",
    "                        # Set up AWS credentials \n",
    "                        aws_access_key_id=AWS_KEY_ID, \n",
    "                         aws_secret_access_key=AWS_SECRET)\n",
    "# List the buckets\n",
    "buckets = s3.list_buckets()\n",
    "\n",
    "# Print the buckets\n",
    "print(buckets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Multiple clients\n",
    "\n",
    "# Generate the boto3 client for interacting with S3 and SNS\n",
    "s3 = boto3.client('s3', region_name='us-east-1',\n",
    "                         aws_access_key_id=AWS_KEY_ID, \n",
    "                         aws_secret_access_key=AWS_SECRET)\n",
    "\n",
    "sns = boto3.client('sns', region_name=\"us-east-1\", \n",
    "                         aws_access_key_id=AWS_KEY_ID, \n",
    "                         aws_secret_access_key=AWS_SECRET)\n",
    "\n",
    "# List S3 buckets and SNS topics\n",
    "buckets = s3.list_buckets()\n",
    "topics = sns.list_topics()\n",
    "\n",
    "# Print out the list of SNS topics\n",
    "print(topics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Creating a bucket\n",
    "\n",
    "import boto3\n",
    "\n",
    "# Create boto3 client to S3\n",
    "s3 = boto3.client('s3', region_name='us-east-1', \n",
    "                         aws_access_key_id=AWS_KEY_ID, \n",
    "                         aws_secret_access_key=AWS_SECRET)\n",
    "\n",
    "# Create the buckets\n",
    "response_staging = s3.create_bucket(Bucket='gim-staging')\n",
    "response_processed = s3.create_bucket(Bucket='gim-processed')\n",
    "response_test = s3.create_bucket(Bucket='gim-test')\n",
    "\n",
    "# Print out the response\n",
    "print(response_staging)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Listing buckets\n",
    "\n",
    "# Get the list_buckets response\n",
    "response = s3.list_buckets()\n",
    "\n",
    "# Iterate over Buckets from .list_buckets() response\n",
    "for bucket in response['Buckets']:\n",
    "  \n",
    "  \t# Print the Name for each bucket\n",
    "    print(bucket['Name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Deleting a bucket\n",
    "\n",
    "# Delete the gim-test bucket\n",
    "s3.delete_bucket(Bucket='gim-test')\n",
    "\n",
    "# Get the list_buckets response\n",
    "response = s3.list_buckets()\n",
    "\n",
    "# Print each Buckets Name\n",
    "for bucket in response['Buckets']:\n",
    "    print(bucket['Name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Deleting multiple buckets\n",
    "\n",
    "# Get the list_buckets response\n",
    "response = s3.list_buckets()\n",
    "\n",
    "# Delete all the buckets with 'gim', create replacements.\n",
    "for bucket in response['Buckets']:\n",
    "  if 'gim' in bucket['Name']:\n",
    "      s3.delete_bucket(Bucket=bucket['Name'])\n",
    "    \n",
    "s3.create_bucket(Bucket='gid-staging')\n",
    "s3.create_bucket(Bucket='gid-processed')\n",
    "  \n",
    "# Print bucket listing after deletion\n",
    "response = s3.list_buckets()\n",
    "for bucket in response['Buckets']:\n",
    "    print(bucket['Name'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Putting files in the cloud\n",
    "\n",
    "# Upload final_report.csv to gid-staging\n",
    "s3.upload_file(Bucket='gid-staging',\n",
    "              # Set filename and key\n",
    "               Filename='final_report.csv', \n",
    "               Key='2019/final_report_01_01.csv')\n",
    "\n",
    "# Get object metadata and print it\n",
    "response = s3.head_object(Bucket='gid-staging', \n",
    "                          Key='2019/final_report_01_01.csv')\n",
    "\n",
    "# Print the size of the uploaded object\n",
    "print(response['ContentLength'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Spring cleaning\n",
    "\n",
    "# List only objects that start with '2018/final_'\n",
    "response = s3.list_objects(Bucket='gid-staging', \n",
    "                           Prefix='2018/final_')\n",
    "\n",
    "# Iterate over the objects\n",
    "if 'Contents' in response:\n",
    "    for obj in response[\"Contents\"]:\n",
    "        # Delete the object\n",
    "        s3.delete_object(Bucket='gid-staging', Key=obj['Key'])\n",
    "\n",
    "# Print the keys of remaining objects in the bucket\n",
    "response = s3.list_objects(Bucket='gid-staging')\n",
    "\n",
    "for obj in response['Contents']:\n",
    "    print(obj['Key'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "2 - Sharing Files Securely."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Uploading a public report\n",
    "\n",
    "# Upload the final_report.csv to gid-staging bucket\n",
    "s3.upload_file(\n",
    "  # Complete the filename\n",
    "  Filename='./final_report.csv', \n",
    "  # Set the key and bucket\n",
    "  Key='2019/final_report_2019_02_20.csv', \n",
    "  Bucket='gid-staging',\n",
    "  # During upload, set ACL to public-read\n",
    "  ExtraArgs = {\n",
    "    'ACL': 'public-read'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Making multiple files public\n",
    "\n",
    "# List only objects that start with '2019/final_'\n",
    "response = s3.list_objects(\n",
    "    Bucket='gid-staging', Prefix='2019/final_')\n",
    "\n",
    "# Iterate over the objects\n",
    "for obj in response['Contents']:\n",
    "\n",
    "    # Give each object ACL of public-read\n",
    "    s3.put_object_acl(Bucket='gid-staging', \n",
    "                      Key=obj['Key'], \n",
    "                      ACL='public-read')\n",
    "    \n",
    "    # Print the Public Object URL for each object\n",
    "    print(\"https://{}.s3.amazonaws.com/{}\".format('gid-staging', obj['Key']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Generating a presigned URL\n",
    "\n",
    "# Generate presigned_url for the uploaded object\n",
    "share_url = s3.generate_presigned_url(\n",
    "  # Specify allowable operations\n",
    "  ClientMethod='get_object',\n",
    "  # Set the expiration time\n",
    "  ExpiresIn=3600,\n",
    "  # Set bucket and shareable object's name\n",
    "  Params={'Bucket': 'gid-staging','Key': 'final_report.csv'}\n",
    ")\n",
    "\n",
    "# Print out the presigned URL\n",
    "print(share_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Opening private files\n",
    "\n",
    "df_list =  [ ] \n",
    "\n",
    "for file in response['Contents']:\n",
    "    # For each file in response load the object from S3\n",
    "    obj = s3.get_object(Bucket='gid-requests', Key=file['Key'])\n",
    "    # Load the object's StreamingBody with pandas\n",
    "    obj_df = pd.read_csv(obj['Body'])\n",
    "    # Append the resulting DataFrame to list\n",
    "    df_list.append(obj_df)\n",
    "\n",
    "# Concat all the DataFrames with pandas\n",
    "df = pd.concat(df_list)\n",
    "\n",
    "# Preview the resulting DataFrame\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Generate HTML file from Pandas \n",
    "\n",
    "# Generate an HTML table with no border and selected columns\n",
    "services_df.to_html('./services_no_border.html',\n",
    "           # Keep specific columns only\n",
    "           columns=['service_name', 'link'],\n",
    "           # Set border\n",
    "           border=0)\n",
    "\n",
    "# Generate an html table with border and all columns.\n",
    "services_df.to_html('./services_border_all_columns.html', \n",
    "           border=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Upload an HTML file to S3\n",
    "\n",
    "# Upload the lines.html file to S3\n",
    "s3.upload_file(Filename='lines.html', \n",
    "               # Set the bucket name\n",
    "               Bucket='datacamp-public', Key='index.html',\n",
    "               # Configure uploaded file\n",
    "               ExtraArgs = {\n",
    "                 # Set proper content type\n",
    "                 'ContentType':'text/html',\n",
    "                 # Set proper ACL\n",
    "                 'ACL': 'public-read'})\n",
    "\n",
    "# Print the S3 Public Object URL for the new file.\n",
    "print(\"http://{}.s3.amazonaws.com/{}\".format('datacamp-public', 'index.html'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Combine daily requests for February\n",
    "\n",
    "df_list = [] \n",
    "\n",
    "# Load each object from s3\n",
    "for file in request_files:\n",
    "    s3_day_reqs = s3.get_object(Bucket='gid-requests', \n",
    "                                Key=file['Key'])\n",
    "    # Read the DataFrame into pandas, append it to the list\n",
    "    day_reqs = pd.read_csv(s3_day_reqs['Body'])\n",
    "    df_list.append(day_reqs)\n",
    "\n",
    "# Concatenate all the DataFrames in the list\n",
    "all_reqs = pd.concat(df_list)\n",
    "\n",
    "# Preview the DataFrame\n",
    "all_reqs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Upload aggregated reports for February\n",
    "\n",
    "# Write agg_df to a CSV and HTML file with no border\n",
    "agg_df.to_csv('./feb_final_report.csv')\n",
    "agg_df.to_html('./feb_final_report.html', \n",
    "               border=0)\n",
    "\n",
    "# Upload the generated CSV to the gid-reports bucket\n",
    "s3.upload_file(Filename='./feb_final_report.csv', \n",
    "               Key='2019/feb/final_report.html', \n",
    "               Bucket='gid-reports',\n",
    "               ExtraArgs = {'ACL': 'public-read'})\n",
    "\n",
    "# Upload the generated HTML to the gid-reports bucket\n",
    "s3.upload_file(Filename='./feb_final_report.html', \n",
    "               Key='2019/feb/final_report.html', \n",
    "               Bucket='gid-reports',\n",
    "               ExtraArgs = {'ContentType': 'text/html', \n",
    "                            'ACL': 'public-read'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Update index to include February\n",
    "\n",
    "# List the gid-reports bucket objects starting with 2019/\n",
    "objects_list = s3.list_objects(Bucket='gid-reports', Prefix='2019/')\n",
    "\n",
    "# Convert the response contents to DataFrame\n",
    "objects_df = pd.DataFrame(objects_list['Contents'])\n",
    "\n",
    "# Create a column \"Link\" that contains Public Object URL\n",
    "base_url = \"http://gid-reports.s3.amazonaws.com/\"\n",
    "objects_df['Link'] = base_url + objects_df['Key']\n",
    "\n",
    "# Preview the resulting DataFrame\n",
    "objects_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Upload the new index\n",
    "\n",
    "# Write objects_df to an HTML file\n",
    "objects_df.to_html('report_listing.html',\n",
    "                   # Set clickable links\n",
    "                   render_links=True,\n",
    "                   # Isolate the columns\n",
    "                   columns=['Link', 'LastModified', 'Size'])\n",
    "\n",
    "# Overwrite index.html key by uploading the new file\n",
    "s3.upload_file(\n",
    "    Filename='./report_listing.html', Key='index.html', \n",
    "    Bucket='gid-reports',\n",
    "    ExtraArgs = {\n",
    "        'ContentType': 'text/html', \n",
    "        'ACL': 'public-read'\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "3 - Reporting and Notifying!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Creating a Topic\n",
    "\n",
    "# Initialize boto3 client for SNS\n",
    "sns = boto3.client('sns', \n",
    "                   region_name='us-east-1', \n",
    "                   aws_access_key_id=AWS_KEY_ID, \n",
    "                   aws_secret_access_key=AWS_SECRET)\n",
    "\n",
    "# Create the city_alerts topic\n",
    "response = sns.create_topic(Name=\"city_alerts\")\n",
    "c_alerts_arn = response['TopicArn']\n",
    "\n",
    "# Re-create the city_alerts topic using a oneliner\n",
    "c_alerts_arn_1 = sns.create_topic(Name='city_alerts')['TopicArn']\n",
    "\n",
    "# Compare the two to make sure they match\n",
    "print(c_alerts_arn == c_alerts_arn_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Creating multiple topics\n",
    "\n",
    "# Create list of departments\n",
    "departments = ['trash', 'streets', 'water']\n",
    "\n",
    "for dept in departments:\n",
    "    # For every department, create a general topic\n",
    "    sns.create_topic(Name=\"{}_general\".format(dept))\n",
    "    \n",
    "    # For every department, create a critical topic\n",
    "    sns.create_topic(Name=\"{}_critical\".format(dept))\n",
    "\n",
    "# Print all the topics in SNS\n",
    "response = sns.list_topics()\n",
    "print(response['Topics'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Deleting multiple topics\n",
    "\n",
    "# Get the current list of topics\n",
    "topics = sns.list_topics()['Topics']\n",
    "\n",
    "for topic in topics:\n",
    "  # For each topic, if it is not marked critical, delete it\n",
    "  if \"critical\" not in topic['TopicArn']:\n",
    "    sns.delete_topic(TopicArn=topic['TopicArn'])\n",
    "    \n",
    "# Print the list of remaining critical topics\n",
    "print(sns.list_topics()['Topics'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Subscribing to topics\n",
    "\n",
    "# Subscribe Elena's phone number to streets_critical topic\n",
    "resp_sms = sns.subscribe(\n",
    "  TopicArn = str_critical_arn, \n",
    "  Protocol='sms', Endpoint=\"+16196777733\")\n",
    "\n",
    "# Print the SubscriptionArn\n",
    "print(resp_sms['SubscriptionArn'])\n",
    "\n",
    "# Subscribe Elena's email to streets_critical topic.\n",
    "resp_email = sns.subscribe(\n",
    "  TopicArn = str_critical_arn, \n",
    "  Protocol='email', Endpoint=\"eblock@sandiegocity.gov\")\n",
    "\n",
    "# Print the SubscriptionArn\n",
    "print(resp_email['SubscriptionArn'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Creating multiple subscriptions\n",
    "\n",
    "# For each email in contacts, create subscription to street_critical\n",
    "for email in contacts['Email']:\n",
    "  sns.subscribe(TopicArn = str_critical_arn,\n",
    "                # Set channel and recipient\n",
    "                Protocol = 'email',\n",
    "                Endpoint =email )\n",
    "\n",
    "# List subscriptions for streets_critical topic, convert to DataFrame\n",
    "response = sns.list_subscriptions_by_topic(\n",
    "  TopicArn = str_critical_arn)\n",
    "subs = pd.DataFrame(response['Subscriptions'])\n",
    "\n",
    "# Preview the DataFrame\n",
    "subs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Deleting multiple subscriptions\n",
    "\n",
    "# List subscriptions for streets_critical topic.\n",
    "response = sns.list_subscriptions_by_topic(\n",
    "  TopicArn = str_critical_arn)\n",
    "\n",
    "# For each subscription, if the protocol is SMS, unsubscribe\n",
    "for sub in response['Subscriptions']:\n",
    "  if sub['Protocol'] == 'sms':\n",
    "\t  sns.unsubscribe(SubscriptionArn=sub['SubscriptionArn'])\n",
    "\n",
    "# List subscriptions for streets_critical topic in one line\n",
    "subs = sns.list_subscriptions_by_topic(\n",
    "  TopicArn=str_critical_arn)['Subscriptions']\n",
    "\n",
    "# Print the subscriptions\n",
    "print(subs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Sending an alert\n",
    "\n",
    "# If there are over 100 potholes, create a message\n",
    "if streets_v_count > 100:\n",
    "  # The message should contain the number of potholes.\n",
    "  message = \"There are {} potholes!\".format(streets_v_count)\n",
    "  # The email subject should also contain number of potholes\n",
    "  subject = \"Latest pothole count is {}\".format(streets_v_count)\n",
    "\n",
    "  # Publish the email to the streets_critical topic\n",
    "  sns.publish(\n",
    "    TopicArn = str_critical_arn,\n",
    "    # Set subject and message\n",
    "    Message = message,\n",
    "    Subject = subject\n",
    "  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Sending a single SMS message\n",
    "\n",
    "# Loop through every row in contacts\n",
    "for idx, row in contacts.iterrows():\n",
    "    #print(row)\n",
    "    # Publish an ad-hoc sms to the user's phone number\n",
    "    response = sns.publish(\n",
    "        # Set the phone number\n",
    "        PhoneNumber = str(row['Phone']),\n",
    "        # The message should include the user's name\n",
    "        Message = 'Hello {}'.format(row['Name'])\n",
    "    )\n",
    "   \n",
    "    print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Creating multi-level topics\n",
    "\n",
    "dept_arns = {} \n",
    "\n",
    "for dept in departments:\n",
    "    # For each deparment, create a critical topic\n",
    "    critical = sns.create_topic(Name=\"{}_critical\".format(dept))\n",
    "    # For each department, create an extreme topic\n",
    "    extreme = sns.create_topic(Name=\"{}_extreme\".format(dept))\n",
    "    # Place the created TopicARNs into a dictionary \n",
    "    dept_arns['{}_critical'.format(dept)] = critical['TopicArn']\n",
    "    dept_arns['{}_extreme'.format(dept)] = extreme['TopicArn']\n",
    "\n",
    "# Print the filled dictionary.\n",
    "print(dept_arns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Different protocols per topic level\n",
    "\n",
    "for index, user_row in contacts.iterrows():\n",
    "    # Get topic names for the users's dept\n",
    "    critical_tname = '{}_critical'.format(user_row['Department'])\n",
    "    extreme_tname = '{}_extreme'.format(user_row['Department'])\n",
    "\n",
    "    # Get or create the TopicArns for a user's department.\n",
    "    critical_arn = sns.create_topic(Name=critical_tname)['TopicArn']\n",
    "    extreme_arn = sns.create_topic(Name=extreme_tname)['TopicArn']\n",
    "\n",
    "    # Subscribe each users email to the critical Topic\n",
    "    sns.subscribe(TopicArn = critical_arn, \n",
    "                  Protocol='email', Endpoint=user_row['Email'])\n",
    "    # Subscribe each users phone number for the extreme Topic\n",
    "    sns.subscribe(TopicArn = extreme_arn, \n",
    "                  Protocol='sms', Endpoint=str(user_row['Phone']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Sending multi-level alerts\n",
    "\n",
    "if vcounts['water'] > 100:\n",
    "  # If over 100 water violations, publish to water_critical\n",
    "  sns.publish(\n",
    "    TopicArn = dept_arns['water_critical'],\n",
    "    Message = \"{} water issues\".format(vcounts['water']),\n",
    "    Subject = \"Help fix water violations NOW!\")\n",
    "\n",
    "if vcounts['water'] > 300:\n",
    "  # If over 300 violations, publish to water_extreme\n",
    "  sns.publish(\n",
    "    TopicArn = dept_arns['water_extreme'],\n",
    "    Message = \"{} violations! RUN!\".format(vcounts['water']),\n",
    "    Subject = \"THIS IS BAD.  WE ARE FLOODING!\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "4 - Pattern Rekognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Cat Detector\n",
    "\n",
    "# Use Rekognition client to detect labels\n",
    "image1_response = rekog.detect_labels(\n",
    "    # Specify the image as an S3Object; Return one label\n",
    "    Image=image1, MaxLabels=1)\n",
    "\n",
    "# Print the labels\n",
    "print(image1_response['Labels'])\n",
    "\n",
    "# Use Rekognition client to detect labels\n",
    "image2_response = rekog.detect_labels(Image=image2, MaxLabels=1)\n",
    "\n",
    "# Print the labels\n",
    "print(image2_response['Labels'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Multiple cat detector\n",
    "\n",
    "# Create an empty counter variable\n",
    "cats_count = 0\n",
    "# Iterate over the labels in the response\n",
    "for label in response['Labels']:\n",
    "    # Find the cat label, look over the detected instances\n",
    "    if label['Name'] == 'Cat':\n",
    "        for instance in label['Instances']:\n",
    "            # Only count instances with confidence > 85\n",
    "            if (instance['Confidence'] > 85):\n",
    "                cats_count += 1\n",
    "# Print count of cats\n",
    "print(cats_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Parking sign reader\n",
    "\n",
    "# Create empty list of words\n",
    "words = []\n",
    "# Iterate over the TextDetections in the response dictionary\n",
    "for text_detection in response['TextDetections']:\n",
    "  \t# If TextDetection type is WORD, append it to words list\n",
    "    if text_detection['Type'] == 'WORD':\n",
    "        # Append the detected text\n",
    "        words.append(text_detection['DetectedText'])\n",
    "# Print out the words list\n",
    "print(words)\n",
    "\n",
    "# <script.py> output:\n",
    "#     ['NO', 'PARKING', '7', 'AM', 'TO', '12', 'NOON', 'MONDAY']\n",
    "\n",
    "\n",
    "# Create empty list of lines\n",
    "lines = []\n",
    "# Iterate over the TextDetections in the response dictionary\n",
    "for text_detection in response['TextDetections']:\n",
    "  \t# If TextDetection type is Line, append it to lines list\n",
    "    if text_detection['Type'] == 'LINE':\n",
    "        # Append the detected text\n",
    "        lines.append(text_detection['DetectedText'])\n",
    "# Print out the words list\n",
    "print(lines)\n",
    "\n",
    "# <script.py> output:\n",
    "#     ['NO PARKING', '7 AM', 'TO', '12 NOON', 'MONDAY']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Detecting language\n",
    "\n",
    "# For each dataframe row\n",
    "for index, row in dumping_df.iterrows():\n",
    "    # Get the public description field\n",
    "    description =dumping_df.loc[index, 'public_description']\n",
    "    if description != '':\n",
    "        # Detect language in the field content\n",
    "        resp = comprehend.detect_dominant_language(Text=description)\n",
    "        # Assign the top choice language to the lang column.\n",
    "        dumping_df.loc[index, 'lang'] = resp['Languages'][0]['LanguageCode']\n",
    "        \n",
    "# Count the total number of spanish posts\n",
    "spanish_post_ct = len(dumping_df[dumping_df.lang == 'es'])\n",
    "# Print the result\n",
    "print(\"{} posts in Spanish\".format(spanish_post_ct))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Translating Get It Done requests\n",
    "\n",
    "for index, row in dumping_df.iterrows():\n",
    "  \t# Get the public_description into a variable\n",
    "    description = dumping_df.loc[index, 'public_description']\n",
    "    if description != '':\n",
    "      \t# Translate the public description\n",
    "        resp = translate.translate_text(\n",
    "            Text=description, \n",
    "            SourceLanguageCode='auto', TargetLanguageCode='en')\n",
    "        # Store original language in original_lang column\n",
    "        dumping_df.loc[index, 'original_lang'] = resp['SourceLanguageCode']\n",
    "        # Store the translation in the translated_desc column\n",
    "        dumping_df.loc[index, 'translated_desc'] = resp['TranslatedText']\n",
    "        \n",
    "# Preview the resulting DataFrame\n",
    "dumping_df = dumping_df[['service_request_id', 'original_lang', 'translated_desc']]\n",
    "dumping_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Getting request sentiment\n",
    "\n",
    "for index, row in dumping_df.iterrows():\n",
    "    \n",
    "  \t# Get the translated_desc into a variable\n",
    "    description = dumping_df.loc[index, 'public_description']\n",
    "    \n",
    "    if description != '':\n",
    "      \t# Get the detect_sentiment response\n",
    "        response = comprehend.detect_sentiment(\n",
    "          Text=description, \n",
    "          LanguageCode='en')\n",
    "        \n",
    "        # Get the sentiment key value into sentiment column\n",
    "        dumping_df.loc[index, 'sentiment'] = response['Sentiment']\n",
    "        \n",
    "# Preview the dataframe\n",
    "dumping_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Scooter community sentiment\n",
    "\n",
    "for index, row in scooter_requests.iterrows():\n",
    "  \t# For every DataFrame row\n",
    "    desc = scooter_requests.loc[index, 'public_description']\n",
    "    if desc != '':\n",
    "      \t# Detect the dominant language\n",
    "        resp = comprehend.detect_dominant_language(Text=desc)\n",
    "        lang_code = resp['Languages'][0]['LanguageCode']\n",
    "        scooter_requests.loc[index, 'lang'] = lang_code\n",
    "        # Use the detected language to determine sentiment\n",
    "        scooter_requests.loc[index, 'sentiment'] = comprehend.detect_sentiment(\n",
    "          Text=desc, \n",
    "            ## use the variable declared above\n",
    "          LanguageCode=lang_code)['Sentiment']\n",
    "          \n",
    "# Perform a count of sentiment by group.\n",
    "counts = scooter_requests.groupby(['sentiment', 'lang']).count()\n",
    "counts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Scooter dispatch\n",
    "\n",
    "# Get topic ARN for scooter notifications\n",
    "topic_arn = sns.create_topic(Name='scooter_notifications')['TopicArn']\n",
    "\n",
    "for index, row in scooter_requests.iterrows():\n",
    "    # Check if notification should be sent\n",
    "    print(row.head)\n",
    "    if (row['sentiment'] == 'NEGATIVE') & (row['img_scooter'] == 1):\n",
    "        # Construct a message to publish to the scooter team.\n",
    "        message = \"Please remove scooter at {}, {}. Description: {}\".format(\n",
    "            row['long'], row['lat'],\n",
    "            row['public_description'])\n",
    "\n",
    "        # Publish the message to the topic!\n",
    "        sns.publish(TopicArn = topic_arn,\n",
    "                    Message = message, \n",
    "                    Subject = \"Scooter Alert\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf-gpu-2.0",
   "language": "python",
   "name": "tf_gpu2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}