f1-2019.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "multiWebScrape.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "vgTaGh7kjVU2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from bs4 import BeautifulSoup\n",
        "from urllib.request import urlopen\n",
        "from google.colab import files\n",
        "import pandas as pd\n",
        "import datetime"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OSi16j4kg-Ag",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Used to extract data from the specified website holding a table of race results.\n",
        "#Function is specifically used to handle data from THAT PARTICULAR WEBSITE but can be altered to target different table data\n",
        "def extractData(soup, element):\n",
        "    rows = soup.find_all(\"tr\", class_=\"ms-table_row\")\n",
        "    row_data = []\n",
        "    for row in rows:\n",
        "      data = row.find_all(\"td\", class_=element)\n",
        "      str_data = str(data)\n",
        "      cleantext = BeautifulSoup(str_data, \"lxml\").get_text()\n",
        "      cleantext = cleantext.strip(\"[]\\n\")\n",
        "      cleantext = cleantext.strip()\n",
        "      row_data.append(cleantext)\n",
        "    \n",
        "    row_data.pop(0)\n",
        "    row_data = row_data[:20]\n",
        "    return row_data\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y8bJeHKfouB5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Used to alter the \"times\" list to add DNF results\n",
        "def addDNF(list):\n",
        "  new_list = []\n",
        "  for each in list:\n",
        "    if each == \"\":\n",
        "      each = \"DNF\"\n",
        "    new_list.append(each)\n",
        "    \n",
        "  return new_list"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RkkAiSdER8YZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def toDateTime(list):\n",
        "  new_list = []\n",
        "  for each in list:\n",
        "    each = each.replace(\".\", \"\")\n",
        "    time_dt = datetime.datetime.strptime(each, \"%M:%S'%f\")\n",
        "    new_list.append(time_dt)\n",
        "  return new_list"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y9Hnk4wtNv2v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def removeNullPoints(list):\n",
        "  new_list = []\n",
        "  for each in list:\n",
        "    if each == \"\":\n",
        "      each = \"0\"\n",
        "    new_list.append(each)\n",
        "  return new_list"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-yLJYhprjmTe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "### MAIN PROGRAM ###\n",
        "#URLs set up for the web scraping\n",
        "main = \"https://www.motorsport.com\"\n",
        "url = \"https://www.motorsport.com/f1/results/\"\n",
        "html = urlopen(url)\n",
        "\n",
        "#Creating the web scraper with BeautifulSoup4\n",
        "soup = BeautifulSoup(html)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oSOzLkSnj3tZ",
        "colab_type": "code",
        "outputId": "d0a6b06d-d3ad-49e2-f880-abd9e5d03b28",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 302
        }
      },
      "source": [
        "#The Main Feature of this Python notebook is that multiple links are searched with BeautifulSoup\n",
        "#These links are added to a list and iterated through by having them concatenated with the root link of the website\n",
        "gp_links = []\n",
        "for link in soup.find_all('a', class_='ms-results-subnav_item'):\n",
        "  print(link.get('href'))\n",
        "  gp_links.append(link.get('href'))"
      ],
      "execution_count": 75,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/f1/results/2019/australian-gp-417660/\n",
            "/f1/results/2019/bahrain-gp/\n",
            "/f1/results/2019/chinese-gp/\n",
            "/f1/results/2019/azerbaijan-gp/\n",
            "/f1/results/2019/spanish-gp-417684/\n",
            "/f1/results/2019/monaco-gp-417689/\n",
            "/f1/results/2019/canadian-gp-417694/\n",
            "/f1/results/2019/french-gp-417699/\n",
            "/f1/results/2019/austrian-gp-417705/\n",
            "/f1/results/2019/british-gp-417710/\n",
            "/f1/results/2019/german-gp/\n",
            "/f1/results/2019/hungarian-gp-417720/\n",
            "/f1/results/2019/belgian-gp-417725/\n",
            "/f1/results/2019/italian-gp-417730/\n",
            "/f1/results/2019/singapore-gp-417735/\n",
            "/f1/results/2019/russian-gp-417740/\n",
            "/f1/results/2019/japanese-gp-417745/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "98cz3yhEZLBg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Need to loop through all existing race results\n",
        "count = 1\n",
        "name = \"race\"\n",
        "\n",
        "for each in gp_links:\n",
        "  url = main + each\n",
        "  html = urlopen(url)\n",
        "  soup = BeautifulSoup(html)\n",
        "\n",
        "  table = createTable(soup)\n",
        "  dataframe = createDataFrame(table)\n",
        "  #download the dataframe as a file\n",
        "  #downloadDT(dataframe, name)\n",
        "  count+=1\n",
        "\n",
        "  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kFz2jVUah9PV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Each of the attributes and its values are collected using the extractData() function and uses the 'element' variable to specify the HTML ID it is contained within\n",
        "def createTable(soup):\n",
        "  element = \"ms-table_cell ms-table_field--\"\n",
        "  names = extractData(soup, element + \"result_driver_id\")\n",
        "  number = extractData(soup, element + \"number\")\n",
        "  position = addDNF(extractData(soup, element + \"pos\"))\n",
        "  car_make = extractData(soup, element + \"car_make\")\n",
        "  times = extractData(soup, element + \"time\")\n",
        "  points = removeNullPoints(extractData(soup, element + \"points\"))\n",
        "\n",
        "  #Dictionary to hold the data and create a Pandas Dataframe\n",
        "  race_results = {\"Position\": position, \"Driver Name\": names, \"Driver No.\": number, \"Team\": car_make, \"Times\": times, \"Points\": points}\n",
        "\n",
        "  return race_results;"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8kgA3igMmhpl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def createDataFrame(table):\n",
        "  results_pd = pd.DataFrame(table)\n",
        "  return results_pd"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rnySHM4qcixq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def downloadDT(dataframe, name):\n",
        "  dataframe.to_csv('filename.csv') \n",
        "  files.download('filename.csv')  "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ywTs8kcEpFfn",
        "colab_type": "code",
        "outputId": "f275375f-53e4-4838-cc9f-abbe89ad3383",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 50
        }
      },
      "source": [
        "#String to DateTime\n",
        "#Tenth and hundreth of the second are stored as microseconds in DateTime Objects\n",
        "#by stripping the decimal point and having '27325 in the microsecond format -> 000000, 000001,...,999999 microseconds\n",
        "time = times[0]\n",
        "time = time.replace(\".\", \"\")\n",
        "print(time)\n",
        "time_dt = datetime.datetime.strptime(time, \"%M:%S'%f\")\n",
        "print(time_dt.microsecond)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1:25'27325\n",
            "273250\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}