get-data-from-web.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_data_from_web():  \n",
    "    '''\n",
    "    Download latest historical and crime datasets from web.\n",
    "    '''\n",
    "    historical_df = pd.read_csv('https://data.london.gov.uk/download/recorded_crime_summary/bf244ffa-326b-49cc-a7df-0e3193abe818/MPS%20LSOA%20Level%20Crime%20%28Historical%29.csv')\n",
    "    recent_df = pd.read_csv('https://data.london.gov.uk/download/recorded_crime_summary/221142dd-f7b2-4209-921e-4de833a82285/MPS%20LSOA%20Level%20Crime%20%28most%20recent%2024%20months%29.csv')\n",
    "\n",
    "    return historical_df, recent_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_data(df:pd.DataFrame, file_name:str, save_directory:str):\n",
    "    '''\n",
    "    After downloading latest dataset, this functions saves the dataset into local storage\n",
    "    '''\n",
    "    \n",
    "    df.to_csv(f'{save_directory}/{file_name}.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_and_save_data():\n",
    "    '''\n",
    "    Downloads and saves the data.\n",
    "\n",
    "    Helper functions: save_data() and download_data_from_web()\n",
    "    '''\n",
    "    \n",
    "    request_time = time.strftime('%Y-%m-%d--%H-%M')\n",
    "\n",
    "    historical_df, recent_df = download_data_from_web()\n",
    "\n",
    "    save_data(df=historical_df, file_name=f'losa-historical--{request_time}', save_directory='data')\n",
    "    save_data(df=recent_df, file_name=f'lsoa-recent--{request_time}', save_directory='data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "download_and_save_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-11-10 18:53:23,900 - INFO - Loading the latest recent file: data/lsoa-recent.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>LSOA Code</th>\n",
       "      <th>LSOA Name</th>\n",
       "      <th>Borough</th>\n",
       "      <th>Major Category</th>\n",
       "      <th>Minor Category</th>\n",
       "      <th>202110</th>\n",
       "      <th>202111</th>\n",
       "      <th>202112</th>\n",
       "      <th>202201</th>\n",
       "      <th>202202</th>\n",
       "      <th>...</th>\n",
       "      <th>202212</th>\n",
       "      <th>202301</th>\n",
       "      <th>202302</th>\n",
       "      <th>202303</th>\n",
       "      <th>202304</th>\n",
       "      <th>202305</th>\n",
       "      <th>202306</th>\n",
       "      <th>202307</th>\n",
       "      <th>202308</th>\n",
       "      <th>202309</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>E01000006</td>\n",
       "      <td>Barking and Dagenham 016A</td>\n",
       "      <td>Barking and Dagenham</td>\n",
       "      <td>Arson and Criminal Damage</td>\n",
       "      <td>Criminal Damage</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>E01000006</td>\n",
       "      <td>Barking and Dagenham 016A</td>\n",
       "      <td>Barking and Dagenham</td>\n",
       "      <td>Burglary</td>\n",
       "      <td>Domestic Burglary</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>E01000006</td>\n",
       "      <td>Barking and Dagenham 016A</td>\n",
       "      <td>Barking and Dagenham</td>\n",
       "      <td>Drug Offences</td>\n",
       "      <td>Drug Trafficking</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>E01000006</td>\n",
       "      <td>Barking and Dagenham 016A</td>\n",
       "      <td>Barking and Dagenham</td>\n",
       "      <td>Drug Offences</td>\n",
       "      <td>Possession of Drugs</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>E01000006</td>\n",
       "      <td>Barking and Dagenham 016A</td>\n",
       "      <td>Barking and Dagenham</td>\n",
       "      <td>Miscellaneous Crimes Against Society</td>\n",
       "      <td>Dangerous Driving</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102227</th>\n",
       "      <td>E01033746</td>\n",
       "      <td>Greenwich 038E</td>\n",
       "      <td>Greenwich</td>\n",
       "      <td>Vehicle Offences</td>\n",
       "      <td>Interfering with a Motor Vehicle</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102228</th>\n",
       "      <td>E01033746</td>\n",
       "      <td>Greenwich 038E</td>\n",
       "      <td>Greenwich</td>\n",
       "      <td>Vehicle Offences</td>\n",
       "      <td>Theft from a Motor Vehicle</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102229</th>\n",
       "      <td>E01033746</td>\n",
       "      <td>Greenwich 038E</td>\n",
       "      <td>Greenwich</td>\n",
       "      <td>Vehicle Offences</td>\n",
       "      <td>Theft or Taking of a Motor Vehicle</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102230</th>\n",
       "      <td>E01033746</td>\n",
       "      <td>Greenwich 038E</td>\n",
       "      <td>Greenwich</td>\n",
       "      <td>Violence Against the Person</td>\n",
       "      <td>Violence with Injury</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102231</th>\n",
       "      <td>E01033746</td>\n",
       "      <td>Greenwich 038E</td>\n",
       "      <td>Greenwich</td>\n",
       "      <td>Violence Against the Person</td>\n",
       "      <td>Violence without Injury</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102232 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        LSOA Code                  LSOA Name               Borough  \\\n",
       "0       E01000006  Barking and Dagenham 016A  Barking and Dagenham   \n",
       "1       E01000006  Barking and Dagenham 016A  Barking and Dagenham   \n",
       "2       E01000006  Barking and Dagenham 016A  Barking and Dagenham   \n",
       "3       E01000006  Barking and Dagenham 016A  Barking and Dagenham   \n",
       "4       E01000006  Barking and Dagenham 016A  Barking and Dagenham   \n",
       "...           ...                        ...                   ...   \n",
       "102227  E01033746             Greenwich 038E             Greenwich   \n",
       "102228  E01033746             Greenwich 038E             Greenwich   \n",
       "102229  E01033746             Greenwich 038E             Greenwich   \n",
       "102230  E01033746             Greenwich 038E             Greenwich   \n",
       "102231  E01033746             Greenwich 038E             Greenwich   \n",
       "\n",
       "                              Major Category  \\\n",
       "0                  Arson and Criminal Damage   \n",
       "1                                   Burglary   \n",
       "2                              Drug Offences   \n",
       "3                              Drug Offences   \n",
       "4       Miscellaneous Crimes Against Society   \n",
       "...                                      ...   \n",
       "102227                      Vehicle Offences   \n",
       "102228                      Vehicle Offences   \n",
       "102229                      Vehicle Offences   \n",
       "102230           Violence Against the Person   \n",
       "102231           Violence Against the Person   \n",
       "\n",
       "                            Minor Category  202110  202111  202112  202201  \\\n",
       "0                          Criminal Damage       0       0       1       2   \n",
       "1                        Domestic Burglary       1       1       1       0   \n",
       "2                         Drug Trafficking       0       0       0       0   \n",
       "3                      Possession of Drugs       1       3       2       5   \n",
       "4                        Dangerous Driving       0       0       0       0   \n",
       "...                                    ...     ...     ...     ...     ...   \n",
       "102227    Interfering with a Motor Vehicle       0       0       0       0   \n",
       "102228          Theft from a Motor Vehicle       0       0       0       0   \n",
       "102229  Theft or Taking of a Motor Vehicle       0       0       0       0   \n",
       "102230                Violence with Injury       1       1       0       0   \n",
       "102231             Violence without Injury       1       2       0       1   \n",
       "\n",
       "        202202  ...  202212  202301  202302  202303  202304  202305  202306  \\\n",
       "0            1  ...       0       0       0       1       1       0       0   \n",
       "1            0  ...       0       0       0       0       1       0       0   \n",
       "2            0  ...       0       0       0       0       1       0       1   \n",
       "3            2  ...       0       0       0       0       0       0       1   \n",
       "4            1  ...       0       0       0       0       0       0       0   \n",
       "...        ...  ...     ...     ...     ...     ...     ...     ...     ...   \n",
       "102227       0  ...       0       0       0       0       0       0       0   \n",
       "102228       1  ...       1       0       0       0       0       0       0   \n",
       "102229       0  ...       0       1       0       2       0       0       1   \n",
       "102230       0  ...       1       0       0       0       0       0       2   \n",
       "102231       2  ...       2       1       1       0       1       0       1   \n",
       "\n",
       "        202307  202308  202309  \n",
       "0            2       0       0  \n",
       "1            0       1       0  \n",
       "2            0       0       0  \n",
       "3            0       2       0  \n",
       "4            0       0       0  \n",
       "...        ...     ...     ...  \n",
       "102227       0       1       0  \n",
       "102228       0       0       0  \n",
       "102229       1       0       0  \n",
       "102230       1       0       0  \n",
       "102231       1       0       3  \n",
       "\n",
       "[102232 rows x 29 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from src.data_preprocessor import load_data\n",
    "\n",
    "load_data(file_type='recent')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}