A-6 Data cleaning(pandas).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>From_To</th>\n",
       "      <th>FlightNumber</th>\n",
       "      <th>RecentDelays</th>\n",
       "      <th>Airline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LoNDon_paris</td>\n",
       "      <td>10045.0</td>\n",
       "      <td>[23, 47]</td>\n",
       "      <td>KLM(!)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MAdrid_miLAN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>&lt;Air France&gt; (12)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>londON_StockhOlm</td>\n",
       "      <td>10065.0</td>\n",
       "      <td>[24, 43, 87]</td>\n",
       "      <td>(British Airways. )</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Budapest_PaRis</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[13]</td>\n",
       "      <td>12. Air France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Brussels_londOn</td>\n",
       "      <td>10085.0</td>\n",
       "      <td>[67, 32]</td>\n",
       "      <td>\"Swiss Air\"</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            From_To  FlightNumber  RecentDelays              Airline\n",
       "0      LoNDon_paris       10045.0      [23, 47]               KLM(!)\n",
       "1      MAdrid_miLAN           NaN            []    <Air France> (12)\n",
       "2  londON_StockhOlm       10065.0  [24, 43, 87]  (British Airways. )\n",
       "3    Budapest_PaRis           NaN          [13]       12. Air France\n",
       "4   Brussels_londOn       10085.0      [67, 32]          \"Swiss Air\""
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'From_To': ['LoNDon_paris', 'MAdrid_miLAN','londON_StockhOlm','Budapest_PaRis', 'Brussels_londOn'],\n",
    "                   'FlightNumber': [10045, np.nan, 10065, np.nan, 10085],\n",
    "                   'RecentDelays': [[23, 47], [], [24, 43, 87], [13], [67, 32]],\n",
    "                   'Airline': ['KLM(!)', '<Air France> (12)', '(British Airways. )','12. Air France', '\"Swiss Air\"']})\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Some values in the the FlightNumber column are missing. These numbers are meant to increase by 10 with each row so 10055 and 10075 need to be put inplace. Fill in these missing numbers and make the column an integer column(instead of a float column)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>From_To</th>\n",
       "      <th>FlightNumber</th>\n",
       "      <th>RecentDelays</th>\n",
       "      <th>Airline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LoNDon_paris</td>\n",
       "      <td>10045.0</td>\n",
       "      <td>[23, 47]</td>\n",
       "      <td>KLM(!)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MAdrid_miLAN</td>\n",
       "      <td>10055.0</td>\n",
       "      <td>[]</td>\n",
       "      <td>&lt;Air France&gt; (12)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>londON_StockhOlm</td>\n",
       "      <td>10065.0</td>\n",
       "      <td>[24, 43, 87]</td>\n",
       "      <td>(British Airways. )</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Budapest_PaRis</td>\n",
       "      <td>10075.0</td>\n",
       "      <td>[13]</td>\n",
       "      <td>12. Air France</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Brussels_londOn</td>\n",
       "      <td>10085.0</td>\n",
       "      <td>[67, 32]</td>\n",
       "      <td>\"Swiss Air\"</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            From_To  FlightNumber  RecentDelays              Airline\n",
       "0      LoNDon_paris       10045.0      [23, 47]               KLM(!)\n",
       "1      MAdrid_miLAN       10055.0            []    <Air France> (12)\n",
       "2  londON_StockhOlm       10065.0  [24, 43, 87]  (British Airways. )\n",
       "3    Budapest_PaRis       10075.0          [13]       12. Air France\n",
       "4   Brussels_londOn       10085.0      [67, 32]          \"Swiss Air\""
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.FlightNumber = df.FlightNumber.fillna(method='ffill')\n",
    "df.FlightNumber[1] = df.FlightNumber[1] + 10\n",
    "df.FlightNumber[3] = df.FlightNumber[3] + 10\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    10045\n",
       "1    10055\n",
       "2    10065\n",
       "3    10075\n",
       "4    10085\n",
       "Name: FlightNumber, dtype: int32"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.FlightNumber.astype('int')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. The From_To column would be better as two separate columns! Split each string on the underscore delimiter _ to give a new temporary DataFrame with the correct values. Assign the correct column names to this temporary DataFrame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LoNDon</td>\n",
       "      <td>paris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>MAdrid</td>\n",
       "      <td>miLAN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>londON</td>\n",
       "      <td>StockhOlm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Budapest</td>\n",
       "      <td>PaRis</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Brussels</td>\n",
       "      <td>londOn</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       From         To\n",
       "0    LoNDon      paris\n",
       "1    MAdrid      miLAN\n",
       "2    londON  StockhOlm\n",
       "3  Budapest      PaRis\n",
       "4  Brussels     londOn"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df = pd.DataFrame()\n",
    "new_df[['From', 'To']] = df['From_To'].str.split('_', expand=True)\n",
    "new_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Notice how the capitalisation of the city names is all mixed up in thistemporary DataFrame. Standardise the strings so that only the first letter isuppercase (e.g. \"londON\" should become \"London\".)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>London</td>\n",
       "      <td>Paris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Madrid</td>\n",
       "      <td>Milan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>London</td>\n",
       "      <td>Stockholm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Budapest</td>\n",
       "      <td>Paris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Brussels</td>\n",
       "      <td>London</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       From         To\n",
       "0    London      Paris\n",
       "1    Madrid      Milan\n",
       "2    London  Stockholm\n",
       "3  Budapest      Paris\n",
       "4  Brussels     London"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df['From'] = new_df['From'].str.capitalize()\n",
    "new_df['To'] = new_df['To'].str.capitalize()\n",
    "new_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Delete the From_To column from df and attach the temporary DataFrame from the previous questions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FlightNumber</th>\n",
       "      <th>RecentDelays</th>\n",
       "      <th>Airline</th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10045.0</td>\n",
       "      <td>[23, 47]</td>\n",
       "      <td>KLM(!)</td>\n",
       "      <td>London</td>\n",
       "      <td>Paris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10055.0</td>\n",
       "      <td>[]</td>\n",
       "      <td>&lt;Air France&gt; (12)</td>\n",
       "      <td>Madrid</td>\n",
       "      <td>Milan</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10065.0</td>\n",
       "      <td>[24, 43, 87]</td>\n",
       "      <td>(British Airways. )</td>\n",
       "      <td>London</td>\n",
       "      <td>Stockholm</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10075.0</td>\n",
       "      <td>[13]</td>\n",
       "      <td>12. Air France</td>\n",
       "      <td>Budapest</td>\n",
       "      <td>Paris</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10085.0</td>\n",
       "      <td>[67, 32]</td>\n",
       "      <td>\"Swiss Air\"</td>\n",
       "      <td>Brussels</td>\n",
       "      <td>London</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   FlightNumber  RecentDelays              Airline      From         To\n",
       "0       10045.0      [23, 47]               KLM(!)    London      Paris\n",
       "1       10055.0            []    <Air France> (12)    Madrid      Milan\n",
       "2       10065.0  [24, 43, 87]  (British Airways. )    London  Stockholm\n",
       "3       10075.0          [13]       12. Air France  Budapest      Paris\n",
       "4       10085.0      [67, 32]          \"Swiss Air\"  Brussels     London"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop('From_To', axis=1, inplace=True)\n",
    "df = pd.concat([df, new_df], axis=1)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. In the RecentDelays column, the values have been entered into the DataFrame as a list. We would like each first value in its own column, each second value in its own column, and so on. If there isn't an Nth value, the value should be NaN.Expand the Series of lists into a DataFrame named delays, rename the columns delay_1, delay_2, etc. and replace the unwanted Recent Delays column in df with delays."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>delay_1</th>\n",
       "      <th>delay_2</th>\n",
       "      <th>delay_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>24.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>87.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>67.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   delay_1  delay_2  delay_3\n",
       "0     23.0     47.0      NaN\n",
       "1      NaN      NaN      NaN\n",
       "2     24.0     43.0     87.0\n",
       "3     13.0      NaN      NaN\n",
       "4     67.0     32.0      NaN"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "delays = pd.DataFrame(df.RecentDelays.values.tolist(), columns=['delay_1', 'delay_2', 'delay_3'])\n",
    "\n",
    "delays\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FlightNumber</th>\n",
       "      <th>Airline</th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "      <th>delay_1</th>\n",
       "      <th>delay_2</th>\n",
       "      <th>delay_3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10045.0</td>\n",
       "      <td>KLM(!)</td>\n",
       "      <td>London</td>\n",
       "      <td>Paris</td>\n",
       "      <td>23.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10055.0</td>\n",
       "      <td>&lt;Air France&gt; (12)</td>\n",
       "      <td>Madrid</td>\n",
       "      <td>Milan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>10065.0</td>\n",
       "      <td>(British Airways. )</td>\n",
       "      <td>London</td>\n",
       "      <td>Stockholm</td>\n",
       "      <td>24.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>87.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10075.0</td>\n",
       "      <td>12. Air France</td>\n",
       "      <td>Budapest</td>\n",
       "      <td>Paris</td>\n",
       "      <td>13.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10085.0</td>\n",
       "      <td>\"Swiss Air\"</td>\n",
       "      <td>Brussels</td>\n",
       "      <td>London</td>\n",
       "      <td>67.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   FlightNumber              Airline      From         To  delay_1  delay_2  \\\n",
       "0       10045.0               KLM(!)    London      Paris     23.0     47.0   \n",
       "1       10055.0    <Air France> (12)    Madrid      Milan      NaN      NaN   \n",
       "2       10065.0  (British Airways. )    London  Stockholm     24.0     43.0   \n",
       "3       10075.0       12. Air France  Budapest      Paris     13.0      NaN   \n",
       "4       10085.0          \"Swiss Air\"  Brussels     London     67.0     32.0   \n",
       "\n",
       "   delay_3  \n",
       "0      NaN  \n",
       "1      NaN  \n",
       "2     87.0  \n",
       "3      NaN  \n",
       "4      NaN  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.drop('RecentDelays', axis=1, inplace=True)\n",
    "\n",
    "df = pd.concat([df, delays], axis=1)\n",
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}