notebooks/scraping_routescanner.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "## Web scraping of RouteScanner\n",
    "\n",
    "A first notebook to web scrape Route Scanner."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [],
   "source": [
    "# Quick and dirty hack because my PyCharm interpreter is not playing nice\n",
    "ewouts_broken_PyCharm = False\n",
    "if ewouts_broken_PyCharm:\n",
    "    import sys\n",
    "    sys.path.append('C:\\\\Users\\\\Ewout\\\\Documents\\\\python_venv\\\\Py310')\n",
    "    sys.path.append('C:\\\\Users\\\\Ewout\\\\Documents\\\\python_venv\\\\Py310\\\\lib\\\\site-packages')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Direct connections\n",
    "\n",
    "In this part a dataframe is created with all direct connections between sea terminals in South America and The Netherlands/Belgium.\n",
    "\n",
    "Data: https://www.routescanner.com/services/direct-connections"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from bs4 import BeautifulSoup\n",
    "import sys\n",
    "import os\n",
    "import itertools\n",
    "import pandas as pd\n",
    "import pickle\n",
    "from numpy import random\n",
    "from time import sleep"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Webscraping"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('BR', 'NL'), ('BR', 'BE'), ('CO', 'NL'), ('CO', 'BE'), ('VE', 'NL'), ('VE', 'BE'), ('SR', 'NL'), ('SR', 'BE'), ('CW', 'NL'), ('CW', 'BE'), ('GY', 'NL'), ('GY', 'BE'), ('GF', 'NL'), ('GF', 'BE'), ('UY', 'NL'), ('UY', 'BE'), ('AR', 'NL'), ('AR', 'BE'), ('CL', 'NL'), ('CL', 'BE'), ('PE', 'NL'), ('PE', 'BE'), ('EC', 'NL'), ('EC', 'BE')]\n"
     ]
    }
   ],
   "source": [
    "# Define origin and destination countries\n",
    "origin = [\"BR\", \"CO\", \"VE\", \"SR\", \"CW\", \"GY\", \"GF\", \"UY\", \"AR\", \"CL\", \"PE\", \"EC\"]\n",
    "destination = [\"NL\", \"BE\"]\n",
    "\n",
    "# Make list with all combinations\n",
    "od_list = list(itertools.product(origin, destination))\n",
    "print(od_list)\n",
    "\n",
    "# Create URL list to scrape\n",
    "urls = [f\"https://www.routescanner.com/services/direct-connections/results?destinationCountries={d}&originCountries={o}&search=advanced\" for o, d in od_list]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "def get_webpage(url, headless=False):\n",
    "    # Instantiate options\n",
    "    opts = Options()\n",
    "    opts.binary_location = \"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\"\n",
    "    opts.add_argument(\"window-size=2880,1920\")\n",
    "    if headless:\n",
    "        opts.headless = True\n",
    "\n",
    "    # Set the location of the webdriver\n",
    "    chrome_service = Service(ChromeDriverManager().install())\n",
    "\n",
    "    # Instantiate a webdriver\n",
    "    driver = webdriver.Chrome(options=opts, service=chrome_service)\n",
    "\n",
    "    # Load the HTML page\n",
    "    driver.get(url)\n",
    "\n",
    "    # Accept cookies\n",
    "    driver.implicitly_wait(2)\n",
    "    driver.find_element(By.CLASS_NAME,\"acceptButton__P2szu\").click()\n",
    "\n",
    "    # Function to get soup from the drivers\n",
    "    def get_soup():\n",
    "        # Wait untill route data is loaded\n",
    "        try:\n",
    "            elem = WebDriverWait(driver, 3).until(EC.presence_of_element_located((By.CLASS_NAME, 'card__hoI9D')))\n",
    "        except:\n",
    "            print('No connections found on this route')\n",
    "\n",
    "        # Parse processed webpage with BeautifulSoup and save in list\n",
    "        return BeautifulSoup(driver.page_source)\n",
    "\n",
    "    # Save the first soup in a new list\n",
    "    soups = [get_soup()]\n",
    "\n",
    "    # Calculate number of results\n",
    "    n_results = int(soups[0].find(\"h6\", class_=\"totalResults__mTaYp\").text.split(' ')[0])\n",
    "\n",
    "    # Get results from other pages\n",
    "    if n_results > 20:\n",
    "        # Calculate number of extra pages\n",
    "        extra_pages = (n_results-1) // 20\n",
    "        # Calculate data for each extra page\n",
    "        for n in range(2, extra_pages+2):\n",
    "            # Go to extra page, get soup and append to soups list\n",
    "            driver.get(f\"{url}&page={n}\")\n",
    "            soups.append(get_soup())\n",
    "\n",
    "    # Close the browser and return soups list and n_results\n",
    "    driver.close()\n",
    "    return soups, n_results"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "39 results across 2 pages\n"
     ]
    },
    {
     "data": {
      "text/plain": "<html lang=\"en\" translate=\"no\" xmlns:fb=\"http://ogp.me/ns/fb#\"><head><meta content=\"width=device-width,initial-scale=1,shrink-to-fit=no\" name=\"viewport\"/><meta content=\"notranslate\" name=\"google\"/><meta charset=\"utf-8\"/><title>Explore direct container connections from one place to another</title><meta content=\"Find the best door-to-door shipping routes by sea and inland. Compare options on lead time and CO2 emissions.\" name=\"description\"/><meta content=\"routescanner, container shipping routes, maritime route, routeplanner, shortsea connections, deepsea connections, rail connections, barge connections\" name=\"keywords\"/><meta content=\"For the best container shipping route comparison, use Routescanner to find the best connection by deepsea, feeder, rail, barge and truck.\" itemprop=\"name\"/><meta content=\"Routescanner – Make better container shipping decisions\" property=\"og:title\"/><meta content=\"Routescanner – Plan your door-to-door container shipping route\" property=\"og:site_name\"/><meta content=\"Find the best door-to-door shipping routes by sea and inland. Compare options on lead time and CO2 emissions.\" property=\"og:description\"/><meta content=\"https://www.routescanner.com/assets/routescanner-thumbnail.jpg\" property=\"og:image\"/><meta content=\"connect-src: https://api.intercom.io https://api.au.intercom.io https://api.eu.intercom.io https://api-iam.intercom.io https://api-iam.eu.intercom.io https://api-iam.au.intercom.io https://api-ping.intercom.io https://nexus-websocket-a.intercom.io wss://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-europe-websocket.intercom.io wss://nexus-europe-websocket.intercom.io https://nexus-australia-websocket.intercom.io wss://nexus-australia-websocket.intercom.io https://uploads.intercomcdn.com https://uploads.intercomcdn.eu https://uploads.au.intercomcdn.com https://uploads.intercomusercontent.com child-src: https://intercom-sheets.com https://www.intercom-reporting.com https://www.youtube.com https://player.vimeo.com https://fast.wistia.net font-src: https://js.intercomcdn.com https://fonts.intercomcdn.com form-action: https://intercom.help https://api-iam.intercom.io https://api-iam.eu.intercom.io https://api-iam.au.intercom.io media-src: https://js.intercomcdn.com img-src: blob: data: https://js.intercomcdn.com https://static.intercomassets.com https://downloads.intercomcdn.com https://downloads.intercomcdn.eu https://downloads.au.intercomcdn.com https://uploads.intercomusercontent.com https://gifs.intercomcdn.com https://video-messages.intercomcdn.com https://messenger-apps.intercom.io https://messenger-apps.eu.intercom.io https://messenger-apps.au.intercom.io https://*.intercom-attachments-1.com https://*.intercom-attachments.eu https://*.au.intercom-attachments.com https://*.intercom-attachments-2.com https://*.intercom-attachments-3.com https://*.intercom-attachments-4.com https://*.intercom-attachments-5.com https://*.intercom-attachments-6.com https://*.intercom-attachments-7.com https://*.intercom-attachments-8.com https://*.intercom-attachments-9.com https://static.intercomassets.eu https://static.au.intercomassets.com style-src: 'unsafe-inline'\" http-equiv=\"Content-Security-Policy\"/><link href=\"/favicon.ico\" rel=\"shortcut icon\"/><link href=\"https://fonts.googleapis.com/css2?family=Lato:wght@300;400;700;900&amp;display=swap\" rel=\"stylesheet\"/><script async=\"\" src=\"https://widget.intercom.io/widget/\" type=\"text/javascript\"></script><script async=\"\" crossorigin=\"anonymous\" integrity=\"sha384-vs3dl3wt0sr1NSmWE/o2E71r7EDd6e3fFXEmu4tyET7uqtZw1URcwd3+DYJQtrMg\" src=\"https://www.gstatic.com/recaptcha/releases/vP4jQKq0YJFzU6e21-BGy3GP/recaptcha__en.js\" type=\"text/javascript\"></script><script async=\"\" src=\"https://www.googleadservices.com/pagead/conversion_async.js\" type=\"text/javascript\"></script><script async=\"\" src=\"https://www.googletagmanager.com/gtag/js?id=G-2L0SPY0SHM&amp;l=dataLayer&amp;cx=c\" type=\"text/javascript\"></script><script async=\"\" src=\"https://diffuser-cdn.app-us1.com/diffuser/diffuser.js\"></script><script async=\"\" src=\"https://static.hotjar.com/c/hotjar-3066325.js?sv=7\" type=\"text/javascript\"></script><script async=\"\" src=\"https://static.hotjar.com/c/hotjar-3066325.js?sv=7\" type=\"text/javascript\"></script><script async=\"\" src=\"https://snap.licdn.com/li.lms-analytics/insight.min.js\" type=\"text/javascript\"></script><script async=\"\" src=\"https://www.google-analytics.com/analytics.js\" type=\"text/javascript\"></script><script async=\"\" src=\"https://www.googletagmanager.com/gtag/js?id=AW-10807702374&amp;l=dataLayer&amp;cx=c\" type=\"text/javascript\"></script><script async=\"\" src=\"https://snap.licdn.com/li.lms-analytics/insight.min.js\" type=\"text/javascript\"></script><script async=\"\" src=\"https://www.google-analytics.com/analytics.js\"></script><script async=\"\" src=\"https://www.googletagmanager.com/gtm.js?id=GTM-KHPF32M\"></script><script src=\"https://secure.page9awry.com/js/218217.js\"></script><noscript><img alt=\"\" src=\"https://secure.page9awry.com/218217.png\" style=\"display:none;\"/></noscript><script async=\"\" src=\"https://www.googletagmanager.com/gtag/js?id=G-AW-10807702374\"></script><script>window.dataLayer = window.dataLayer || []\n    function gtag(){dataLayer.push(arguments)}\n    gtag('js', new Date())\n    gtag('config', 'AW-10807702374')</script><script>if (document.cookie.length > 0) {\n      let cStart = document.cookie.indexOf(`RTSdisclaimer=`)\n      if (cStart !== -1) {\n        cStart = cStart + `RTSdisclaimer=`.length\n        let cEnd = document.cookie.indexOf(';', cStart)\n        if (cEnd === -1) {\n          cEnd = document.cookie.length\n        }\n        const cookieSet = unescape(document.cookie.substring(cStart, cEnd)) === '1'\n        if (cookieSet) {\n          //- <!-- Google Tag Manager -->\n          (function(w,d,s,l,i){\n            w[l]=w[l]||[];\n            w[l].push({\n            'gtm.start': new Date().getTime(),event:'gtm.js'\n            });\n            var f=d.getElementsByTagName(s)[0],\n                j=d.createElement(s),\n                dl=l!='dataLayer'?'&l='+l:'';\n            j.async=true;\n            j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;\n            f.parentNode.insertBefore(j,f);\n          })(window,document,'script','dataLayer','GTM-KHPF32M');\n          //- <!-- End Google Tag Manager -->\n\n          // if cookie settings are accepted, set ga\n          (function(i, s, o, g, r, a, m) {\n            i['GoogleAnalyticsObject'] = r; i[r] = i[r] || function() {\n              (i[r].q = i[r].q || []).push(arguments)\n            }, i[r].l = 1 * new Date(); a = s.createElement(o),\n              m = s.getElementsByTagName(o)[0]; a.async = 1; a.src = g; m.parentNode.insertBefore(a, m)\n          })(window, document, 'script', 'https://www.google-analytics.com/analytics.js', 'ga')\n          ga('create', 'UA-180471736-1', 'auto')\n        }\n      }\n    } else {\n      var ga = undefined;\n    }</script><script defer=\"defer\" src=\"/runtime.e92c935df3b9a4076203.js\"></script><script defer=\"defer\" src=\"/6528.0a93b1be9519fcebf355.js\"></script><script defer=\"defer\" src=\"/app.511b169b28abf74f40c9.js\"></script><link href=\"/6528.5bf76cfdc45407134c1c.css\" rel=\"stylesheet\"/><link href=\"/app.d5465480da02f77eda70.css\" rel=\"stylesheet\"/><script async=\"\" src=\"https://static.hotjar.com/c/hotjar-3070458.js?sv=6\"></script><meta content=\"A7bG5hJ4XpMV5a3V1wwAR0PalkFSxLOZeL9D/YBYdupYUIgUgGhfVJ1zBFOqGybb7gRhswfJ+AmO7S2rNK2IOwkAAAB7eyJvcmlnaW4iOiJodHRwczovL3d3dy5nb29nbGV0YWdtYW5hZ2VyLmNvbTo0NDMiLCJmZWF0dXJlIjoiUHJpdmFjeVNhbmRib3hBZHNBUElzIiwiZXhwaXJ5IjoxNjY5NzY2Mzk5LCJpc1RoaXJkUGFydHkiOnRydWV9\" http-equiv=\"origin-trial\"/><script id=\"google-recaptcha-v3\" src=\"https://www.google.com/recaptcha/api.js?render=6Ld4PvIfAAAAAOE_01RWd8Zk1rnIpNRE0_Vis9Kb\"></script><script async=\"\" src=\"https://googleads.g.doubleclick.net/pagead/viewthroughconversion/10807702374/?random=1666206989954&amp;cv=9&amp;fst=1666206989954&amp;num=1&amp;bg=ffffff&amp;guid=ON&amp;resp=GooglemKTybQhCsO&amp;u_h=1440&amp;u_w=2560&amp;u_ah=1440&amp;u_aw=2497&amp;u_cd=24&amp;u_his=2&amp;u_tz=120&amp;u_java=false&amp;u_nplug=5&amp;u_nmime=2&amp;gtm=2oaah0&amp;sendb=1&amp;ig=1&amp;data=event%3Dgtag.config&amp;frm=0&amp;url=https%3A%2F%2Fwww.routescanner.com%2Fservices%2Fdirect-connections%2Fresults%3FdestinationCountries%3DNL%26originCountries%3DBR%26search%3Dadvanced&amp;tiba=Routescanner%20%E2%80%93%20Make%20better%20container%20shipping%20decisions&amp;auid=1218558230.1666206988&amp;hn=www.googleadservices.com&amp;async=1&amp;rfmt=3&amp;fmt=4\" type=\"text/javascript\"></script><link href=\"/9575.4d0d6c23375e3b9efc52.css\" rel=\"stylesheet\" type=\"text/css\"/><link href=\"/6873.7af07570fa63add35b69.css\" rel=\"stylesheet\" type=\"text/css\"/><link href=\"/9114.88cfe25e89b1ef3ed8aa.css\" rel=\"stylesheet\" type=\"text/css\"/><style data-emotion=\"css\" data-s=\"\"></style></head><body><noscript><iframe height=\"0\" src=\"https://www.googletagmanager.com/ns.html?id=GTM-KHPF32M\" style=\"display:none;visibility:hidden\" width=\"0\"></iframe></noscript><div id=\"routescanner\"><div class=\"app__qr3h+\"><header class=\"header__1wEHX\"><div class=\"headerContent__qiyjb\"><button aria-expanded=\"false\" aria-haspopup=\"true\" class=\"mobileNavToggleButton__zaaH8 closed__z8xCq iconDbsIconsBold19__T2O2N\" role=\"toggle\"></button><nav class=\"mobileNav__JHbGe notVisible__IWHRn\"><div class=\"mobileMenuHeader__iRvC5\"><div class=\"opacityLayer__Z5pZL undefined\"></div></div><hr class=\"headerBorderLine__7stZI\"/><ul class=\"mobileMenu__6M1hi\"><li class=\"mobileNavItem__8hX+z\">Solutions<span class=\"iconDbsIconsBold32__CnnLb dropdownIcon__pbVfd\"></span></li><li class=\"mobileNavItem__8hX+z\">Join as an operator</li><li class=\"mobileNavItem__8hX+z\">Ecosystem<span class=\"iconDbsIconsBold32__CnnLb dropdownIcon__pbVfd\"></span></li><li class=\"mobileNavItem__8hX+z\">Company<span class=\"iconDbsIconsBold32__CnnLb dropdownIcon__pbVfd\"></span></li></ul><div class=\"buttonGroup__-izfC\"><form action=\"/api/auth0\" method=\"post\"><button class=\"secondary__clc5- medium__j0QW4 undefined\" type=\"submit\">Login</button></form><button class=\"primary__PIeCF medium__j0QW4 undefined\" type=\"button\">Register</button></div></nav><div class=\"routescannerLogo__mbKiy\"><svg aria-label=\"[title + description]\" class=\"link__xN8Ly\" role=\"img\" viewbox=\"0 0 370.21 33.74\" width=\"100\" xmlns=\"http://www.w3.org/2000/svg\"><title>Routescanner Logo</title><g data-name=\"Laag 2\"><g data-name=\"Laag 1\"><path d=\"M15.74 14.27h-2.7q-5.74 0-5.74 5v13.92H0V7h7.06v3.32a6.28 6.28 0 012.68-2.5A8.33 8.33 0 0113.47 7h2.27zM30.58 33.52A12.85 12.85 0 0121.19 30q-3.56-3.49-3.55-9.94t3.55-10a12.85 12.85 0 019.39-3.48A12.93 12.93 0 0140 10.12q3.56 3.49 3.55 10T40 30a12.93 12.93 0 01-9.44 3.48m0-21a4.91 4.91 0 00-4 1.85c-1 1.24-1.52 3.13-1.52 5.69s.51 4.45 1.52 5.67a4.91 4.91 0 004 1.82 5 5 0 004-1.82q1.52-1.83 1.52-5.67c0-2.56-.5-4.45-1.5-5.69a4.92 4.92 0 00-4.05-1.85M70.55 33.19h-7.21v-2.32q-2.55 2.65-7.25 2.65-4.41 0-6.76-2.49A9.13 9.13 0 0147 24.51V7h7.3v16.14c0 3 1.32 4.45 4 4.45a4.67 4.67 0 003.65-1.46 5.89 5.89 0 001.33-4.08V7h7.3zM89.75 33.1a52.26 52.26 0 01-5.74.28 11 11 0 01-3.62-.5 3.85 3.85 0 01-2.09-1.63 6.77 6.77 0 01-.9-2.42 21.31 21.31 0 01-.21-3.32V12.85h-3.8V7h3.8V0h7.35v7h5.54v5.88h-5.54v12.25a2.31 2.31 0 00.46 1.61 2.27 2.27 0 001.61.43l3.14-.17zM105 33.52q-6.16 0-9.58-3.7A13.77 13.77 0 0192 20.1a13.36 13.36 0 013.63-9.74 12.44 12.44 0 019.31-3.72q6 0 9.37 3.86t3.34 10.27v.9h-18a6.75 6.75 0 001.31 4.19 4.73 4.73 0 004 1.74 5.09 5.09 0 003.68-1.19 3.79 3.79 0 001.3-2.23l7.21.14a8.56 8.56 0 01-.83 2.85 14.13 14.13 0 01-2 2.94 9.22 9.22 0 01-3.75 2.46 15.29 15.29 0 01-5.62 1m-5.1-16.74h9.86a4.59 4.59 0 00-1.51-3.36 4.91 4.91 0 00-3.2-1.09 5.1 5.1 0 00-3.48 1.25 4.7 4.7 0 00-1.64 3.2M202.84 15a2.34 2.34 0 00-1.14-2.07 5.73 5.73 0 00-3.08-.73 5.93 5.93 0 00-2.73.57 1.89 1.89 0 00-1.11 1.8c0 .73.55 1.26 1.66 1.59a26.45 26.45 0 004 .81 38.9 38.9 0 014.7.9 8 8 0 014 2.37 6.61 6.61 0 011.66 4.71 8.8 8.8 0 01-.54 3.11 8.1 8.1 0 01-5.62 4.94 20.46 20.46 0 01-6 .78 19 19 0 01-4.67-.52 9.21 9.21 0 01-3.27-1.42 12.24 12.24 0 01-2-1.85 5.66 5.66 0 01-1.14-2c-.22-.72-.37-1.28-.45-1.69a6 6 0 01-.12-1.14h7.35q0 3.29 4.74 3.28a6.57 6.57 0 003.13-.67 2.14 2.14 0 001.24-2 1.58 1.58 0 00-.62-1.26 3.45 3.45 0 00-1.35-.68 18.3 18.3 0 00-2-.38l-1.42-.19a35.68 35.68 0 01-4.1-.76 15.23 15.23 0 01-3.25-1.28 5.61 5.61 0 01-2.42-2.4 7.87 7.87 0 01-.8-3.72 6.91 6.91 0 013.15-6 14 14 0 018.08-2.16 17.63 17.63 0 015.08.66 10 10 0 013.31 1.59 7.83 7.83 0 011.89 2.06 6.65 6.65 0 01.88 2.06A8.74 8.74 0 01210 15zM231 23.5h7.3a11.05 11.05 0 01-3.74 7.64 12.72 12.72 0 01-8.3 2.6 13.16 13.16 0 01-9.48-3.43q-3.61-3.45-3.61-9.89t3.56-10a14.38 14.38 0 0117.8-.85 10.55 10.55 0 013.72 7.7h-7.34q-.53-4.45-4.75-4.45a4.86 4.86 0 00-4 1.82c-1 1.22-1.47 3.12-1.47 5.72s.49 4.4 1.47 5.64a4.74 4.74 0 003.89 1.85 5.15 5.15 0 003.25-1.05 4.8 4.8 0 001.7-3.3M241.34 15.77q.22-8.91 11.66-8.91a19.39 19.39 0 015.86.78 8.33 8.33 0 013.65 2.11 8.36 8.36 0 011.7 2.7 8.41 8.41 0 01.5 2.94v14.13a13.42 13.42 0 00.14 1.83 13.13 13.13 0 00.29 1.49l.14.57h-7.73V31q-2.61 2.7-7.63 2.7a10.26 10.26 0 01-6.81-2.2 7.49 7.49 0 01-2.63-6.05 7.21 7.21 0 01.9-3.65 6.13 6.13 0 012.37-2.37 17.27 17.27 0 012.64-1.21 10.72 10.72 0 012.34-.54l8.63-1.24v-.9a2.72 2.72 0 00-.95-2.25 5.52 5.52 0 00-3.41-.78q-4.6 0-4.6 3.22zm16 7.59v-1.85q-4.55.57-6.35.95a4.08 4.08 0 00-2.21 1 2.68 2.68 0 00-.83 2 2.94 2.94 0 00.83 2 3.59 3.59 0 002.73.9 5.75 5.75 0 004.39-1.54 5 5 0 001.44-3.53M293.35 33.41H286V17.29a4.55 4.55 0 00-1.07-3.29 3.91 3.91 0 00-3-1.11 5 5 0 00-3.77 1.49 6 6 0 00-1.45 4.34v14.69h-7.3V7.19h7.21v2.56a9.29 9.29 0 017.21-2.89 9.73 9.73 0 017 2.37 8.48 8.48 0 012.49 6.45zM321.84 33.41h-7.35V17.29a4.54 4.54 0 00-1.06-3.29 3.92 3.92 0 00-3-1.11 5 5 0 00-3.76 1.49 6 6 0 00-1.45 4.34v14.69H298V7.19h7.2v2.56a9.31 9.31 0 017.21-2.89 9.76 9.76 0 017 2.37 8.51 8.51 0 012.48 6.45zM338.25 33.74q-6.17 0-9.58-3.7a13.77 13.77 0 01-3.41-9.72 13.39 13.39 0 013.62-9.74 12.46 12.46 0 019.32-3.72q6 0 9.37 3.86T350.91 21v.9h-18a6.8 6.8 0 001.3 4.2 4.76 4.76 0 004.06 1.73 5 5 0 003.67-1.19 3.81 3.81 0 001.31-2.23l7.2.14a8.33 8.33 0 01-.83 2.85 13.38 13.38 0 01-2 2.94 9.25 9.25 0 01-3.74 2.46 15.29 15.29 0 01-5.62 1m-5.12-16.69H343a4.62 4.62 0 00-1.52-3.36 4.89 4.89 0 00-3.22-1.09 5.19 5.19 0 00-3.49 1.25 4.69 4.69 0 00-1.63 3.2M370.21 14.49h-2.71q-5.73 0-5.73 5v13.92h-7.31V7.19h7.07v3.32a6.15 6.15 0 012.68-2.46 8.27 8.27 0 013.72-.86h2.28zM147.83 13.66a2.78 2.78 0 10-2.78 2.78 2.78 2.78 0 002.78-2.78M124.58 11.21h13.28v4.9h-13.28zM152.25 11.21h27.98v4.9h-27.98zM164.83 28.32a2.78 2.78 0 10-2.78 2.78 2.78 2.78 0 002.78-2.78M124.58 25.87h30.28v4.9h-30.28zM169.25 25.87h10.98v4.9h-10.98z\"></path></g></g></svg></div><nav class=\"nav__5Id9l\"><div class=\"navItem__-nZ9T\">Solutions<span class=\"iconDbsIconsBold32__CnnLb dropdownIcon__pbVfd\"></span></div><div class=\"navItem__-nZ9T\">Join as an operator</div><div class=\"navItem__-nZ9T\">Ecosystem<span class=\"iconDbsIconsBold32__CnnLb dropdownIcon__pbVfd\"></span></div><div class=\"navItem__-nZ9T\">Company<span class=\"iconDbsIconsBold32__CnnLb dropdownIcon__pbVfd\"></span></div></nav><div class=\"buttonGroup__-izfC\"><form action=\"/api/auth0\" method=\"post\"><button class=\"secondary__clc5- medium__j0QW4 undefined\" type=\"submit\">Login</button></form><button class=\"primary__PIeCF medium__j0QW4 undefined\" type=\"button\">Register</button></div></div></header><div class=\"wrapper__v8OjR\"><aside class=\"sidebar__UW6Ie\"><h1 class=\"title__BeYkf\">Explore direct connections</h1><div class=\"stickyHeader__-njwQ\"><div class=\"searchSummary__W20jn\"><div class=\"verticalLine__UheW9\"><div class=\"routes__-RTA9\"><p title=\"BR\">BR</p></div><div class=\"routes__-RTA9\"><p title=\"NL\">NL</p></div></div><button class=\"secondary__clc5- medium__j0QW4 fullWidth__1T7Xj undefined\" type=\"button\">Edit search</button></div><button class=\"primary__PIeCF medium__j0QW4 filterButton__-LtAy\" type=\"button\">Filter overview </button><div class=\"resultsHeader__cyv1j\"><h6 class=\"totalResults__mTaYp\">39 direct connections found</h6></div></div><div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"4b606865-62cd-4c6d-a3f1-44707f6f978a\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>Hapag-Lloyd</h6><small class=\"locodes__tdIbs\">BRPEC - NLRTM</small><p class=\"serviceCode__igooW\">ECX-EUROPE EAST COAST EXPRESS</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>10 - 11 days</div><div><span class=\"iconDbsIconsOutlines94__lOHWq\"></span>1 time per week</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"4da514eb-34a8-4885-92e5-92195a6f0134\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>Maersk</h6><small class=\"locodes__tdIbs\">BRPNG - NLRTM</small><p class=\"serviceCode__igooW\">NEOSAMBA</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>21 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"e8e22615-9ec4-4745-8ed1-9261e749ba8b\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>Hamburg Sud</h6><small class=\"locodes__tdIbs\">BRPNG - NLRTM</small><p class=\"serviceCode__igooW\">SOUTH AMERICA EAST COAST -- NORTH EUROPE SLING 1 (SAEC1)</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>21 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"f5ff394d-e5c0-4e93-9f8d-8b147aecd3dc\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>Hapag-Lloyd</h6><small class=\"locodes__tdIbs\">BRRIO - NLRTM</small><p class=\"serviceCode__igooW\">ECX-EUROPE EAST COAST EXPRESS</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>16 - 18 days</div><div><span class=\"iconDbsIconsOutlines94__lOHWq\"></span>1 time per week</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"e849a51f-f99c-4dee-b645-3f6a17b32031\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>Hapag-Lloyd</h6><small class=\"locodes__tdIbs\">BRPNG - NLRTM</small><p class=\"serviceCode__igooW\">ECX-EUROPE EAST COAST EXPRESS</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>19 - 21 days</div><div><span class=\"iconDbsIconsOutlines94__lOHWq\"></span>1 time per week</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"e00c8d1a-278c-4478-a30a-a3609a80d1c4\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>CMA CGM</h6><small class=\"locodes__tdIbs\">BRFOR - NLRTM</small><p class=\"serviceCode__igooW\">North Europe French Guiana North Brazil</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>17 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"8c9bdef5-5026-4041-9c61-e64eab6ae3c1\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>MSC</h6><small class=\"locodes__tdIbs\">BRSSZ - NLRTM</small><p class=\"serviceCode__igooW\">NWC TO SAEC - STRING I</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>19 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"54b18314-b5df-4857-9f8a-4679e20c5a9f\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>CMA CGM</h6><small class=\"locodes__tdIbs\">BRPNG - NLRTM</small><p class=\"serviceCode__igooW\">SOUTH AMERICA FR. NORTH EUR. PLATE SLING</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>21 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"4872cc65-9a80-4b16-bd61-580a8002bfe7\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>MSC</h6><small class=\"locodes__tdIbs\">BRSSZ - NLRTM</small><p class=\"serviceCode__igooW\">NWC TO SAEC - STRING I</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>19 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"cca33c0f-131c-4409-869e-c6da52e43d08\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>MSC</h6><small class=\"locodes__tdIbs\">BRPNG - NLRTM</small><p class=\"serviceCode__igooW\">NWC TO SAEC - STRING I</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>19 - 21 days</div><div><span class=\"iconDbsIconsOutlines94__lOHWq\"></span>1 time per week</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"43c702e0-3b88-4a2e-a430-9679b55e8b82\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>Hamburg Sud</h6><small class=\"locodes__tdIbs\">BRSSZ - NLRTM</small><p class=\"serviceCode__igooW\">SOUTH AMERICA EAST COAST -- NORTH EUROPE SLING 1 (SAEC1)</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>19 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"f6f56bee-2254-45a6-b155-2d962469ee5b\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>Hapag-Lloyd</h6><small class=\"locodes__tdIbs\">BRRIG - NLRTM</small><p class=\"serviceCode__igooW\">ECX-EUROPE EAST COAST EXPRESS</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>22 - 25 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"67f079ac-21b1-4397-9d9f-224040f03b4c\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>CMA CGM</h6><small class=\"locodes__tdIbs\">BRSSZ - NLRTM</small><p class=\"serviceCode__igooW\">SOUTH AMERICA FR. NORTH EUR. PLATE SLING</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>19 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"7dbf4f0e-dbfe-47c3-9eb8-41730bfb1226\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>MSC</h6><small class=\"locodes__tdIbs\">BRITJ - NLRTM</small><p class=\"serviceCode__igooW\">NWC TO SAEC - STRING I</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>20 - 22 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"4917e74c-f195-4103-92af-1e3346c89e82\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>Maersk</h6><small class=\"locodes__tdIbs\">BRSSZ - NLRTM</small><p class=\"serviceCode__igooW\">NEOSAMBA</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>19 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"74897942-87a2-4e5b-9f60-f7c11030657f\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>MSC</h6><small class=\"locodes__tdIbs\">BRSSZ - NLRTM</small><p class=\"serviceCode__igooW\">NWC TO SAEC - STRING I</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>17 - 19 days</div><div><span class=\"iconDbsIconsOutlines94__lOHWq\"></span>1 time per week</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"f6f6523d-6b85-4eed-89ad-49b662a9424f\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>Maersk</h6><small class=\"locodes__tdIbs\">BRPNG - NLRTM</small><p class=\"serviceCode__igooW\">NEOSAMBA</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>21 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"4e63e3a7-453f-41e1-b0ac-739317761c0d\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>Hamburg Sud</h6><small class=\"locodes__tdIbs\">BRPNG - NLRTM</small><p class=\"serviceCode__igooW\">SOUTH AMERICA EAST COAST -- NORTH EUROPE SLING 1 (SAEC1)</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>21 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"4abc3ad3-c411-44a4-a00f-0d503293e5f4\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>MSC</h6><small class=\"locodes__tdIbs\">BRRIO - NLRTM</small><p class=\"serviceCode__igooW\">NWC TO SAEC - STRING I</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>18 days</div></div></div><div class=\"card__hoI9D\" data-modality=\"deepsea\" data-selected=\"false\" id=\"98355861-015e-4330-a8fe-4fa75de48565\"><div><div class=\"menuMore__VGg9y\"></div></div><h6>MSC</h6><small class=\"locodes__tdIbs\">BRPNG - NLRTM</small><p class=\"serviceCode__igooW\">NWC TO SAEC - STRING I</p><div class=\"times__xgwdu\"><div><span class=\"iconDbsIconsOutlines106__hZsy9\"></span>21 days</div></div></div></div><div class=\"pagination__kodD9\"><span class=\"page__JDipi iconDbsIconsOutlines115__uX6aK deactivated__cj8FO\"></span><div class=\"page__JDipi currentPage__adEYi\">1</div><div class=\"page__JDipi false\">2</div><span class=\"page__JDipi iconDbsIconsOutlines116__ZQpo5\"></span></div><div class=\"total__70syY\">0 - 20 of the 39 results</div></aside><div class=\"wrapper__FVq9t\"><div class=\"mapboxgl-map\" style=\"height: 100%; width: 100%;\"><div class=\"mapboxgl-canary\" style=\"visibility: hidden;\"></div><div class=\"mapboxgl-canvas-container mapboxgl-interactive mapboxgl-touch-drag-pan mapboxgl-touch-zoom-rotate\"><canvas aria-label=\"Map\" class=\"mapboxgl-canvas\" height=\"1855\" role=\"region\" style=\"width: 2063px; height: 1237px;\" tabindex=\"0\" width=\"3094\"></canvas></div><div class=\"mapboxgl-control-container\"><div class=\"mapboxgl-ctrl-top-left\"></div><div class=\"mapboxgl-ctrl-top-right\"></div><div class=\"mapboxgl-ctrl-bottom-left\"><div class=\"mapboxgl-ctrl\" style=\"display: block;\"><a aria-label=\"Mapbox logo\" class=\"mapboxgl-ctrl-logo\" href=\"https://www.mapbox.com/\" rel=\"noopener nofollow\" target=\"_blank\"></a></div></div><div class=\"mapboxgl-ctrl-bottom-right\"><div class=\"mapboxgl-ctrl mapboxgl-ctrl-attrib\"><button aria-label=\"Toggle attribution\" class=\"mapboxgl-ctrl-attrib-button\" title=\"Toggle attribution\"></button><div class=\"mapboxgl-ctrl-attrib-inner\" role=\"list\"><a aria-label=\"Mapbox\" href=\"https://www.mapbox.com/about/maps/\" role=\"listitem\" target=\"_blank\" title=\"Mapbox\">© Mapbox</a> <a aria-label=\"OpenStreetMap\" href=\"https://www.openstreetmap.org/about/\" role=\"listitem\" target=\"_blank\" title=\"OpenStreetMap\">© OpenStreetMap</a> <a aria-label=\"Map feedback\" class=\"mapbox-improve-map\" href=\"https://apps.mapbox.com/feedback/?owner=mapbox&amp;id=light-v9&amp;access_token=pk.eyJ1IjoibmF2aWdhdGUtY29ubmVjdGlvbnMiLCJhIjoiY2p6bDU2eWFxMDU4YzNubDlrYThkenZkbyJ9.9oVhHWn2C7xxl-ipyrUU4g\" rel=\"noopener nofollow\" role=\"listitem\" target=\"_blank\" title=\"Map feedback\">Improve this map</a></div></div></div></div><div class=\"filters__sgEot\"><div class=\"css-2b097c-container\"><span aria-atomic=\"false\" aria-live=\"polite\" aria-relevant=\"additions text\" class=\"css-7pg0cj-a11yText\"></span><div class=\"react-select__control css-wo76l-control\"><div class=\"react-select__value-container react-select__value-container--is-multi css-1hwfws3\"><div class=\"react-select__placeholder css-ipl185-placeholder\">Modalities</div><input aria-autocomplete=\"list\" class=\"css-62g3xt-dummyInput\" id=\"react-select-2-input\" readonly=\"\" tabindex=\"0\" value=\"\"/></div><div class=\"react-select__indicators css-1wy0on6\"><span class=\"react-select__indicator-separator css-1fs5mrq-indicatorSeparator\"></span><div aria-hidden=\"true\" class=\"react-select__indicator react-select__dropdown-indicator css-hy23g1-indicatorContainer\"><span class=\"iconDbsIconsFilled123__gV7WV false\"></span></div></div></div><div><input name=\"modalities\" type=\"hidden\"/></div></div><div class=\"css-2b097c-container\"><span aria-atomic=\"false\" aria-live=\"polite\" aria-relevant=\"additions text\" class=\"css-7pg0cj-a11yText\"></span><div class=\"react-select__control css-wo76l-control\"><div class=\"react-select__value-container react-select__value-container--is-multi css-1hwfws3\"><div class=\"react-select__placeholder css-15y6nme-placeholder\">Operators</div><input aria-autocomplete=\"list\" class=\"css-62g3xt-dummyInput\" id=\"react-select-3-input\" readonly=\"\" tabindex=\"0\" value=\"\"/></div><div class=\"react-select__indicators css-1wy0on6\"><span class=\"react-select__indicator-separator css-1hyfx7x\"></span><div aria-hidden=\"true\" class=\"react-select__indicator react-select__dropdown-indicator css-hy23g1-indicatorContainer\"><span class=\"iconDbsIconsFilled123__gV7WV false\"></span></div></div></div><div><input name=\"operators\" type=\"hidden\"/></div></div><div class=\"css-2b097c-container\"><span aria-atomic=\"false\" aria-live=\"polite\" aria-relevant=\"additions text\" class=\"css-7pg0cj-a11yText\"></span><div class=\"react-select__control css-wo76l-control\"><div class=\"react-select__value-container css-1hwfws3\"><div class=\"react-select__placeholder css-15y6nme-placeholder\">Transit times</div><input aria-autocomplete=\"list\" class=\"css-62g3xt-dummyInput\" id=\"react-select-4-input\" readonly=\"\" tabindex=\"0\" value=\"\"/></div><div class=\"react-select__indicators css-1wy0on6\"><span class=\"react-select__indicator-separator css-1hyfx7x\"></span><div aria-hidden=\"true\" class=\"react-select__indicator react-select__dropdown-indicator css-hy23g1-indicatorContainer\"><span class=\"iconDbsIconsFilled123__gV7WV false\"></span></div></div></div><input name=\"transitTimes\" type=\"hidden\" value=\"\"/></div></div><div class=\"missingRoutes__N563g\"><div class=\"missingRoutesInner__nbdtk\">!</div></div> </div></div></div></div></div><script>var blogPosts = !{blogPosts}</script><script>_linkedin_partner_id = \"dummy-value\"\n    window._linkedin_data_partner_ids = window._linkedin_data_partner_ids || []\n    window._linkedin_data_partner_ids.push(_linkedin_partner_id)</script><script>(() => {\n      var s = document.getElementsByTagName(\"script\")[0]\n      var b = document.createElement(\"script\")\n      b.type = \"text/javascript\"\n      b.async = true\n      b.src = \"https://snap.licdn.com/li.lms-analytics/insight.min.js\"\n      s.parentNode.insertBefore(b, s)\n    })()</script><script id=\"\" type=\"text/javascript\">(function(a,d,e,f,g,b,c){a.visitorGlobalObjectAlias=f;a[a.visitorGlobalObjectAlias]=a[a.visitorGlobalObjectAlias]||function(){(a[a.visitorGlobalObjectAlias].q=a[a.visitorGlobalObjectAlias].q||[]).push(arguments)};a[a.visitorGlobalObjectAlias].l=(new Date).getTime();b=d.createElement(\"script\");b.src=e;b.async=!0;c=d.getElementsByTagName(\"script\")[0];c.parentNode.insertBefore(b,c)})(window,document,\"https://diffuser-cdn.app-us1.com/diffuser/diffuser.js\",\"vgo\");vgo(\"setAccount\",\"650412485\");\nvgo(\"setTrackByDefault\",!0);vgo(\"process\");</script>\n<script id=\"\" type=\"text/javascript\">(function(a,c,e,f,d,b){a.hj=a.hj||function(){(a.hj.q=a.hj.q||[]).push(arguments)};a._hjSettings={hjid:3070458,hjsv:6};d=c.getElementsByTagName(\"head\")[0];b=c.createElement(\"script\");b.async=1;b.src=e+a._hjSettings.hjid+f+a._hjSettings.hjsv;d.appendChild(b)})(window,document,\"https://static.hotjar.com/c/hotjar-\",\".js?sv\\x3d\");</script><div><div class=\"grecaptcha-badge\" data-style=\"bottomright\" style=\"width: 256px; height: 60px; display: block; transition: right 0.3s ease 0s; position: fixed; bottom: 14px; right: -186px; box-shadow: gray 0px 0px 5px; border-radius: 2px; overflow: hidden;\"><div class=\"grecaptcha-logo\"><iframe frameborder=\"0\" height=\"60\" name=\"a-8x7ub0eh5jsu\" role=\"presentation\" sandbox=\"allow-forms allow-popups allow-same-origin allow-scripts allow-top-navigation allow-modals allow-popups-to-escape-sandbox\" scrolling=\"no\" src=\"https://www.google.com/recaptcha/api2/anchor?ar=1&amp;k=6Ld4PvIfAAAAAOE_01RWd8Zk1rnIpNRE0_Vis9Kb&amp;co=aHR0cHM6Ly93d3cucm91dGVzY2FubmVyLmNvbTo0NDM.&amp;hl=en&amp;v=vP4jQKq0YJFzU6e21-BGy3GP&amp;size=invisible&amp;cb=kxzqn6ij4ffj\" title=\"reCAPTCHA\" width=\"256\"></iframe></div><div class=\"grecaptcha-error\"></div><textarea class=\"g-recaptcha-response\" id=\"g-recaptcha-response-100000\" name=\"g-recaptcha-response\" style=\"width: 250px; height: 40px; border: 1px solid rgb(193, 193, 193); margin: 10px 25px; padding: 0px; resize: none; display: none;\"></textarea></div><iframe style=\"display: none;\"></iframe></div><iframe height=\"0\" src=\"https://auth.routescanner.com/authorize?audience=https%3A%2F%2Fapi.routescanner.com&amp;scope=openid%20profile%20email%20read%3Acurrent_user%20update%3Acurrent_user_metadata%20feature%3Aquotes%20read%3Ausers%20admin%3Acollaborations%20read%3AconnectivitySummary%20offline_access&amp;client_id=fLxQLwcPNs8B6YEC423ohJ5rjyPyLvxr&amp;redirect_uri=https%3A%2F%2Fwww.routescanner.com&amp;response_type=code&amp;response_mode=web_message&amp;state=Y2JjaXN6S2YySUNTeH5RWGNLLThyYk04d042M1RqdG5MZS1QQTZPcXhCRA%3D%3D&amp;nonce=UXh0dUF0YnJWVTZpRn45flNjd1hVdlc5QzhUa2tqVWEyQTZDUExqTTNSNQ%3D%3D&amp;code_challenge=Agz2VWfpiVtSz_WVKq4SDAxsynRpXED5BxJj3dmH3Ts&amp;code_challenge_method=S256&amp;prompt=none&amp;auth0Client=eyJuYW1lIjoiYXV0aDAtcmVhY3QiLCJ2ZXJzaW9uIjoiMS4xMS4wIn0%3D\" style=\"display: none;\" width=\"0\"></iframe></body></html>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A test run with a single webpage\n",
    "soups1, n1 = get_webpage(urls[0])\n",
    "print(f\"{n1} results across {len(soups1)} pages\")\n",
    "soups1[0]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Extract data from HTML"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "# Returns route data for list of routes\n",
    "def get_route_data(soups):\n",
    "    # Create empty list for the route data\n",
    "    route_data = []\n",
    "\n",
    "    # Iterate through soup pages\n",
    "    for soup in soups:\n",
    "        # Create a list with all route cards\n",
    "        routes = soup.find_all(\"div\", class_=\"card__hoI9D\")\n",
    "\n",
    "        # For each route, find the company, origin, destination, service code, duration and if known frequency\n",
    "        for route in routes:\n",
    "            c = route.find(\"h6\").text\n",
    "            o, d = route.find(\"small\", class_=\"locodes__tdIbs\").text.split(' - ')\n",
    "            s = route.find(\"p\", class_=\"serviceCode__igooW\").text\n",
    "            t = route.find(\"div\", class_=\"times__xgwdu\").find_all(\"div\")\n",
    "            t = [i.text for i in t]\n",
    "            route_data.append([c, o, d, s, *t])\n",
    "    return route_data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created a DataFrame with 39 entries\n"
     ]
    },
    {
     "data": {
      "text/plain": "       Company Origin Destination  \\\n0  Hapag-Lloyd  BRPEC       NLRTM   \n1       Maersk  BRPNG       NLRTM   \n2  Hamburg Sud  BRPNG       NLRTM   \n3  Hapag-Lloyd  BRRIO       NLRTM   \n4  Hapag-Lloyd  BRPNG       NLRTM   \n\n                                        Service Code      Duration  \\\n0                      ECX-EUROPE EAST COAST EXPRESS  10 - 11 days   \n1                                           NEOSAMBA       21 days   \n2  SOUTH AMERICA EAST COAST -- NORTH EUROPE SLING...       21 days   \n3                      ECX-EUROPE EAST COAST EXPRESS  16 - 18 days   \n4                      ECX-EUROPE EAST COAST EXPRESS  19 - 21 days   \n\n         Frequency  \n0  1 time per week  \n1             None  \n2             None  \n3  1 time per week  \n4  1 time per week  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Company</th>\n      <th>Origin</th>\n      <th>Destination</th>\n      <th>Service Code</th>\n      <th>Duration</th>\n      <th>Frequency</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>Hapag-Lloyd</td>\n      <td>BRPEC</td>\n      <td>NLRTM</td>\n      <td>ECX-EUROPE EAST COAST EXPRESS</td>\n      <td>10 - 11 days</td>\n      <td>1 time per week</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Maersk</td>\n      <td>BRPNG</td>\n      <td>NLRTM</td>\n      <td>NEOSAMBA</td>\n      <td>21 days</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Hamburg Sud</td>\n      <td>BRPNG</td>\n      <td>NLRTM</td>\n      <td>SOUTH AMERICA EAST COAST -- NORTH EUROPE SLING...</td>\n      <td>21 days</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Hapag-Lloyd</td>\n      <td>BRRIO</td>\n      <td>NLRTM</td>\n      <td>ECX-EUROPE EAST COAST EXPRESS</td>\n      <td>16 - 18 days</td>\n      <td>1 time per week</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>Hapag-Lloyd</td>\n      <td>BRPNG</td>\n      <td>NLRTM</td>\n      <td>ECX-EUROPE EAST COAST EXPRESS</td>\n      <td>19 - 21 days</td>\n      <td>1 time per week</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get route data sample\n",
    "data1 = get_route_data(soups1)\n",
    "\n",
    "# Define dict keys / dataframe columns\n",
    "columns = [\"Company\", \"Origin\", \"Destination\", \"Service Code\", \"Duration\", \"Frequency\"]\n",
    "\n",
    "# Create dataframe from the data sample\n",
    "df1 = pd.DataFrame(data1, columns=columns)\n",
    "print(f\"Created a DataFrame with {df1.index.size} entries\")\n",
    "df1.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Run on all routes"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "       Company Origin Destination  \\\n0          MSC  BRRIG       NLRTM   \n1  Hamburg Sud  BRPNG       NLRTM   \n2          MSC  BRPNG       NLRTM   \n3  Hapag-Lloyd  BRPNG       NLRTM   \n4      CMA CGM  BRSSZ       NLRTM   \n\n                                        Service Code      Duration  \\\n0                             NWC TO SAEC - STRING I       25 days   \n1  SOUTH AMERICA EAST COAST -- NORTH EUROPE SLING...       21 days   \n2                             NWC TO SAEC - STRING I       21 days   \n3                      ECX-EUROPE EAST COAST EXPRESS  20 - 21 days   \n4           SOUTH AMERICA FR. NORTH EUR. PLATE SLING       19 days   \n\n         Frequency  \n0             None  \n1             None  \n2  1 time per week  \n3  1 time per week  \n4             None  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Company</th>\n      <th>Origin</th>\n      <th>Destination</th>\n      <th>Service Code</th>\n      <th>Duration</th>\n      <th>Frequency</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>MSC</td>\n      <td>BRRIG</td>\n      <td>NLRTM</td>\n      <td>NWC TO SAEC - STRING I</td>\n      <td>25 days</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>Hamburg Sud</td>\n      <td>BRPNG</td>\n      <td>NLRTM</td>\n      <td>SOUTH AMERICA EAST COAST -- NORTH EUROPE SLING...</td>\n      <td>21 days</td>\n      <td>None</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>MSC</td>\n      <td>BRPNG</td>\n      <td>NLRTM</td>\n      <td>NWC TO SAEC - STRING I</td>\n      <td>21 days</td>\n      <td>1 time per week</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Hapag-Lloyd</td>\n      <td>BRPNG</td>\n      <td>NLRTM</td>\n      <td>ECX-EUROPE EAST COAST EXPRESS</td>\n      <td>20 - 21 days</td>\n      <td>1 time per week</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>CMA CGM</td>\n      <td>BRSSZ</td>\n      <td>NLRTM</td>\n      <td>SOUTH AMERICA FR. NORTH EUR. PLATE SLING</td>\n      <td>19 days</td>\n      <td>None</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The data can be loaded from a pickle, so the script doesn't need to re-run every time.\n",
    "# Set use_pickle1 to False if you want to scrape all the data again.\n",
    "use_pickle1 = True\n",
    "\n",
    "if use_pickle1:\n",
    "    route_df = pd.read_pickle(\"../pickles/routescanner_routes_between_ports.pickle\")\n",
    "\n",
    "else:\n",
    "    # Empty list for data and len\n",
    "    route_data = []\n",
    "    combinations = len(urls)\n",
    "\n",
    "    for n, url in enumerate(urls):\n",
    "        # Scrape the webpage\n",
    "        soups, _ = get_webpage(url, headless=True)\n",
    "        # Get all the route data, process them and add\n",
    "        new_route_data = get_route_data(soups)\n",
    "        route_data = route_data + new_route_data\n",
    "        # Nice print message\n",
    "        print(f\"Done with {n+1}/{combinations} webpages, {len(new_route_data)} new routes found (total {len(route_data)})\")\n",
    "\n",
    "    # Create a dataframe from all the route data\n",
    "    route_df = pd.DataFrame(route_data, columns=columns)\n",
    "route_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "# Save as Pickle and CSV\n",
    "route_df.to_pickle(\"../pickles/routescanner_routes_between_ports.pickle\")\n",
    "route_df.to_csv(\"../data/routescanner_routes_between_ports.csv\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Planned routes\n",
    "\n",
    "Scrapes planned connections from Routescanner and collects them in a DataFrame.\n",
    "\n",
    "Data: https://www.routescanner.com/voyages"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25 origin ports: ['BRRIG' 'BRPNG' 'BRSSZ' 'BRPEC' 'BRRIO' 'BRITJ' 'BRNAT' 'BRSSA' 'BRFOR'\n",
      " 'COCTG' 'COTRB' 'COSMR' 'SRPBM' 'CWWIL' 'GFDDC' 'UYMVD' 'ARBUE' 'CLSAI'\n",
      " 'CLVAP' 'CLCNL' 'PECLL' 'PEPAI' 'ECPSJ' 'ECGYE' 'ECPBO']\n",
      "4 destination ports: ['NLRTM' 'BEANR' 'NLVLI' 'BEZEE']\n",
      "100 combinations of ports\n"
     ]
    }
   ],
   "source": [
    "# Create list of origin and destination ports\n",
    "o_ports = route_df[\"Origin\"].unique()\n",
    "d_ports = route_df[\"Destination\"].unique()\n",
    "print(f\"{len(o_ports)} origin ports: {o_ports}\\n{len(d_ports)} destination ports: {d_ports}\")\n",
    "\n",
    "# Make list with all combinations\n",
    "od_ports = list(itertools.product(o_ports, d_ports))\n",
    "print(f\"{len(od_ports)} combinations of ports\")"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-10-19\n"
     ]
    }
   ],
   "source": [
    "# Use today's date, by default\n",
    "from datetime import date\n",
    "today = date.today()\n",
    "print(today)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.routescanner.com/voyages?limit=100&from=BRRIG&fromType=locode&originsNearby=1&to=NLRTM&toType=locode&destinationsNearby=1&departure=2022-10-19&sort=transfers&modalities=sea\n"
     ]
    }
   ],
   "source": [
    "# Define search criteria\n",
    "limit = 100\n",
    "date = today             # Change this to anything later than today, in \"YYYY-MM-DD\" format\n",
    "sort_on = \"transfers\"    # Sort options (str): \"emission_co2\", \"transfers\", \"arrival\", \"duration\"\n",
    "modalities = [\"sea\"]     # Modalities options (list): [\"sea\", \"barge\", \"rail\", \"truck\"]\n",
    "modalaties_string = \"%2C\".join(modalities)\n",
    "\n",
    "# Create list of URLs to scrape\n",
    "def generate_urls2(od_ports=od_ports):\n",
    "    return [f\"https://www.routescanner.com/voyages?limit={limit}&from={o_port}&fromType=locode&originsNearby=1&to={d_port}&toType=locode&destinationsNearby=1&departure={date}&sort={sort_on}&modalities={modalaties_string}\" for o_port, d_port in od_ports]\n",
    "\n",
    "urls2 = generate_urls2()\n",
    "print(urls2[0])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('https://www.routescanner.com/voyages?limit=100&from=BRRIG&fromType=locode&originsNearby=1&to=NLRTM&toType=locode&destinationsNearby=1&departure=2022-10-19&sort=transfers&modalities=sea', ('BRRIG', 'NLRTM'))\n"
     ]
    }
   ],
   "source": [
    "urls12 = generate_urls2(od_ports)\n",
    "combis = list(zip(urls12, od_ports))\n",
    "print(combis[0])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [],
   "source": [
    "def get_webpages2(od_ports, headless=True):\n",
    "    urls = generate_urls2(od_ports)\n",
    "    u_od_zip = list(zip(urls, od_ports))\n",
    "\n",
    "    # Instantiate options\n",
    "    opts = Options()\n",
    "    #opts.binary_location = \"/usr/bin/google-chrome\"\n",
    "    opts.binary_location = \"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe\"\n",
    "    if headless:\n",
    "        opts.headless = True\n",
    "    opts.add_argument('--remote-debugging-port=9222')\n",
    "    opts.add_argument('--disable-gpu')\n",
    "    opts.add_argument(\"--window-size=2880,2160\")\n",
    "\n",
    "    # Some options to make Chrome (hopefully) more\n",
    "    opts.add_argument('--disable-blink-features=AutomationControlled')\n",
    "    opts.add_experimental_option('useAutomationExtension', False)\n",
    "    opts.add_experimental_option(\"excludeSwitches\", [\"enable-automation\"])\n",
    "\n",
    "    # Set the location of the webdriver\n",
    "    os.environ[\"WDM_PROGRESS_BAR\"] = '0'\n",
    "    chrome_service = Service(ChromeDriverManager().install())\n",
    "    already_got = False\n",
    "\n",
    "    def start_browser(url):\n",
    "        print(\"(Re)starting browser\")\n",
    "        # Instantiate a webdriver\n",
    "        driver = webdriver.Chrome(options=opts, service=chrome_service)\n",
    "        # Load the HTML page\n",
    "        driver.get(url)\n",
    "        already_got = True\n",
    "        # Accept cookies\n",
    "        driver.implicitly_wait(5)\n",
    "        driver.find_element(By.CLASS_NAME,\"acceptButton__P2szu\").click()\n",
    "        return driver\n",
    "\n",
    "    driver = start_browser(urls[0])\n",
    "\n",
    "    soups = []\n",
    "    sleeptime = 20\n",
    "    driver_age = 0\n",
    "    for n, (url, od) in enumerate(u_od_zip):\n",
    "        i = 1\n",
    "        while True: # Keep trying until successfully scraped\n",
    "            try:\n",
    "                # If the driver too old, first restart the browser\n",
    "                if driver_age >= 4:\n",
    "                    driver.quit()\n",
    "                    driver_age = 0\n",
    "                    driver = start_browser(url)\n",
    "                # Load the HTML page, if not done earlier by a browser start\n",
    "                if not already_got:\n",
    "                    driver.get(url)\n",
    "                already_got = False\n",
    "                # Wait until route data is loaded\n",
    "                elem = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CLASS_NAME, \"voyages__NVlid \")))\n",
    "                # Parse processed webpage with BeautifulSoup and append to list\n",
    "                soups.append((BeautifulSoup(driver.page_source, features=\"html.parser\"), od))\n",
    "                # Increase the driver age and decrease the sleeptime\n",
    "                driver_age += 1\n",
    "                sleeptime = max(7, sleeptime - 3)\n",
    "                # Print and sleep\n",
    "                print(f\"Scraped route {n+1}/{len(u_od_zip)}, took {i} tries (sleeptime {sleeptime})\")\n",
    "                sleep(random.uniform(sleeptime, sleeptime+2))\n",
    "            except:\n",
    "                if i > 6 or sleeptime > 120:\n",
    "                    print(f\"Stopped after {i} sequential failed attempts. {n} routes successfully collected.\")\n",
    "                    return soups\n",
    "                sleeptime = sleeptime + 10\n",
    "                print(f\"Failed attempt {i} on route {n+1}, trying again (sleeptime {sleeptime})\")\n",
    "\n",
    "                i += 1\n",
    "                # Restart browser after 2 unsuccessful tries, or if the driver gets old\n",
    "                if i >= 3 and i % 3 == 0 or driver_age >= 4:\n",
    "                    driver.quit()\n",
    "                    driver_age = 0\n",
    "                    sleep(random.uniform(sleeptime, sleeptime+2))\n",
    "                    driver = start_browser(url)\n",
    "                else:\n",
    "                    sleep(random.uniform(sleeptime, sleeptime+2))\n",
    "                continue\n",
    "            #print(f\"Done {n}, iteration {i}, sleeping {sleeptime}\")\n",
    "            break\n",
    "\n",
    "    # Close the browser and return soups list and n_results\n",
    "    driver.close()\n",
    "    driver.quit()\n",
    "    return soups"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [],
   "source": [
    "# Returns route data for list of routes\n",
    "def get_route_data2(soups):\n",
    "    # Create empty list for the route data\n",
    "    route_data = []\n",
    "\n",
    "    for soup, od in soups:\n",
    "        # Create a list with all route cards\n",
    "        routes = soup.find_all(\"li\", class_=\"voyage voyage__e0GtV\")\n",
    "        o, d = od\n",
    "\n",
    "        # For each route, find the company, origin, destination, service code, duration and if known frequency\n",
    "        for route in routes:\n",
    "            lead = route.find(\"li\", class_=\"voyageMetaColumn__qkBJv\").text\n",
    "\n",
    "            small = route.find_all(\"li\", class_=\"voyageMetaColumn__qkBJv voyageMetaColumnSmall__QMXjZ\", limit=2)\n",
    "            transfers, co2 = small[0].text, small[1].text\n",
    "\n",
    "            time_fields = route.find_all(\"li\", class_=\"voyageMetaColumn__qkBJv smallerText__ED-90\", limit=2)\n",
    "            dep = [span.text for span in time_fields[0].find_all(\"span\") if span.text != ' - ']\n",
    "            arr = [span.text for span in time_fields[1].find_all(\"span\") if span.text != ' - ']\n",
    "\n",
    "            c = route.find_all(\"span\", class_=\"carrierNames__Dn5bC\")\n",
    "            companies = [n.text for n in c]\n",
    "\n",
    "            # Split the rout in different legs, and get the ports and companies for each\n",
    "            legs = route.find_all(\"ul\", class_=\"leg__9DiZ9\")\n",
    "            ports = []\n",
    "            for leg in legs:\n",
    "                # Get the origin, in between, and destination ports for each leg\n",
    "                o_port = [leg.find(\"li\", class_=\"origin__Vu5JN\").text]\n",
    "                stop_ports = leg.find_all(\"span\", class_=\"stopLocode__V9cgh\")\n",
    "                stop_ports = [n.text for n in stop_ports]\n",
    "                d_port = [leg.find(\"li\", class_=\"destination__lN9tN\").text]\n",
    "                # Combine all ports in one list, and append that to the ports list\n",
    "                ports.append(o_port + stop_ports + d_port)\n",
    "\n",
    "            # Validate number of legs collected:\n",
    "            if len(ports) != int(transfers)+1:\n",
    "                print(\"Warning: Ports data may be incomplete\")\n",
    "\n",
    "            # Add everything to the route_data list\n",
    "            route_data.append([lead, transfers, o, d, dep, arr, co2, ports, companies])\n",
    "    return route_data"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(Re)starting browser\n",
      "Scraped route 1/1, took 1 tries (sleeptime 17)\n",
      "Created a DataFrame with 6 entries\n"
     ]
    },
    {
     "data": {
      "text/plain": "  Lead time Transfers Origin Destination                    Departure  \\\n0   28 days         1  BRRIG       NLRTM  [21 OCT,  02:00, UTC-03:00]   \n1   28 days         1  BRRIG       NLRTM  [27 OCT,  06:00, UTC-03:00]   \n2   23 days         1  BRRIG       NLRTM  [26 OCT,  18:02, UTC-03:00]   \n3   33 days         1  BRRIG       NLRTM  [27 OCT,  06:00, UTC-03:00]   \n4   32 days         2  BRRIG       NLRTM  [22 OCT,  08:00, UTC-03:00]   \n5   35 days         2  BRRIG       NLRTM   [5 NOV,  08:13, UTC-03:00]   \n\n                       Arrival kg CO2e/TEU  \\\n0  [18 NOV,  00:00, UTC+01:00]        1010   \n1  [23 NOV,  13:00, UTC+01:00]        1060   \n2  [18 NOV,  00:00, UTC+01:00]        1690   \n3  [29 NOV,  07:00, UTC+01:00]        1785   \n4  [23 NOV,  00:01, UTC+01:00]        1175   \n5  [10 DEC,  09:26, UTC+01:00]        1805   \n\n                                               Ports  \\\n0            [[RIG, IOA, SSZ], [SSZ, PTM, LGP, RTM]]   \n1  [[RIG, ITJ], [ITJ, PNG, SSZ, RIO, SSA, PEC, PT...   \n2            [[RIG, PNG], [PNG, SSZ, PTM, LGP, RTM]]   \n3       [[RIG, ITJ, SSZ, RIO, CTG], [CTG, CAU, RTM]]   \n4  [[RIG, ITJ, PNG, SSZ, RIO, SSA, PEC, PTM], [PT...   \n5  [[RIG, ITJ, PNG, SSZ, RIO, SSA, SUA, CTB], [CT...   \n\n                                            Carriers  \n0  [Hapag-Lloyd, Maersk, CMA CGM, Hamburg Sud, Ma...  \n1       [Hapag-Lloyd, Maersk, ONE, Hapag-Lloyd, MSC]  \n2                     [CMA CGM, Hamburg Sud, Maersk]  \n3  [Hapag-Lloyd, Maersk, MSC, ONE, COSCO, Hapag-L...  \n4  [Hapag-Lloyd, Hapag-Lloyd, Hapag-Lloyd, W.E.C....  \n5                                         [MSC, MSC]  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Lead time</th>\n      <th>Transfers</th>\n      <th>Origin</th>\n      <th>Destination</th>\n      <th>Departure</th>\n      <th>Arrival</th>\n      <th>kg CO2e/TEU</th>\n      <th>Ports</th>\n      <th>Carriers</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>28 days</td>\n      <td>1</td>\n      <td>BRRIG</td>\n      <td>NLRTM</td>\n      <td>[21 OCT,  02:00, UTC-03:00]</td>\n      <td>[18 NOV,  00:00, UTC+01:00]</td>\n      <td>1010</td>\n      <td>[[RIG, IOA, SSZ], [SSZ, PTM, LGP, RTM]]</td>\n      <td>[Hapag-Lloyd, Maersk, CMA CGM, Hamburg Sud, Ma...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>28 days</td>\n      <td>1</td>\n      <td>BRRIG</td>\n      <td>NLRTM</td>\n      <td>[27 OCT,  06:00, UTC-03:00]</td>\n      <td>[23 NOV,  13:00, UTC+01:00]</td>\n      <td>1060</td>\n      <td>[[RIG, ITJ], [ITJ, PNG, SSZ, RIO, SSA, PEC, PT...</td>\n      <td>[Hapag-Lloyd, Maersk, ONE, Hapag-Lloyd, MSC]</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>23 days</td>\n      <td>1</td>\n      <td>BRRIG</td>\n      <td>NLRTM</td>\n      <td>[26 OCT,  18:02, UTC-03:00]</td>\n      <td>[18 NOV,  00:00, UTC+01:00]</td>\n      <td>1690</td>\n      <td>[[RIG, PNG], [PNG, SSZ, PTM, LGP, RTM]]</td>\n      <td>[CMA CGM, Hamburg Sud, Maersk]</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>33 days</td>\n      <td>1</td>\n      <td>BRRIG</td>\n      <td>NLRTM</td>\n      <td>[27 OCT,  06:00, UTC-03:00]</td>\n      <td>[29 NOV,  07:00, UTC+01:00]</td>\n      <td>1785</td>\n      <td>[[RIG, ITJ, SSZ, RIO, CTG], [CTG, CAU, RTM]]</td>\n      <td>[Hapag-Lloyd, Maersk, MSC, ONE, COSCO, Hapag-L...</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>32 days</td>\n      <td>2</td>\n      <td>BRRIG</td>\n      <td>NLRTM</td>\n      <td>[22 OCT,  08:00, UTC-03:00]</td>\n      <td>[23 NOV,  00:01, UTC+01:00]</td>\n      <td>1175</td>\n      <td>[[RIG, ITJ, PNG, SSZ, RIO, SSA, PEC, PTM], [PT...</td>\n      <td>[Hapag-Lloyd, Hapag-Lloyd, Hapag-Lloyd, W.E.C....</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>35 days</td>\n      <td>2</td>\n      <td>BRRIG</td>\n      <td>NLRTM</td>\n      <td>[5 NOV,  08:13, UTC-03:00]</td>\n      <td>[10 DEC,  09:26, UTC+01:00]</td>\n      <td>1805</td>\n      <td>[[RIG, ITJ, PNG, SSZ, RIO, SSA, SUA, CTB], [CT...</td>\n      <td>[MSC, MSC]</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get some soup\n",
    "soup_od_2 = get_webpages2([od_ports[0]], headless=True)\n",
    "\n",
    "# Get route data sample\n",
    "data2 = get_route_data2(soup_od_2)\n",
    "\n",
    "# Define dict keys / dataframe columns\n",
    "columns2 = [\"Lead time\", \"Transfers\", \"Origin\", \"Destination\", \"Departure\", \"Arrival\", \"kg CO2e/TEU\", \"Ports\", \"Carriers\"]\n",
    "\n",
    "# Create dataframe from the data sample\n",
    "df2 = pd.DataFrame(data2, columns=columns2)\n",
    "print(f\"Created a DataFrame with {df2.index.size} entries\")\n",
    "df2"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DataFrame has 1394 entries\n"
     ]
    },
    {
     "data": {
      "text/plain": "  Lead time Transfers Origin Destination                    Departure  \\\n0   11 days         1  BRFOR       NLRTM  [21 OCT,  02:57, UTC-03:00]   \n1   23 days         1  BRFOR       BEANR  [27 OCT,  00:00, UTC-03:00]   \n2   16 days         2  BRFOR       BEANR  [21 OCT,  02:57, UTC-03:00]   \n3   22 days         2  BRFOR       BEANR  [21 OCT,  02:57, UTC-03:00]   \n4   18 days         2  BRFOR       BEANR  [21 OCT,  02:57, UTC-03:00]   \n\n                       Arrival kg CO2e/TEU  \\\n0   [1 NOV,  06:00, UTC+01:00]         700   \n1  [18 NOV,  22:00, UTC+01:00]         860   \n2   [5 NOV,  22:00, UTC+01:00]         755   \n3  [11 NOV,  18:00, UTC+01:00]         760   \n4   [8 NOV,  06:00, UTC+01:00]         780   \n\n                                            Ports  \\\n0                   [[FOR, PEC], [PEC, PTM, RTM]]   \n1    [[FOR, NAT, VGO, LGP, RTM], [RTM, BRV, ANR]]   \n2       [[FOR, PEC], [PEC, PTM, RTM], [RTM, ANR]]   \n3       [[FOR, PEC], [PEC, PTM, RTM], [RTM, ANR]]   \n4  [[FOR, PEC], [PEC, PTM, RTM, LGP], [LGP, ANR]]   \n\n                                        Carriers  \n0                             [Hapag-Lloyd, MSC]  \n1     [CMA CGM, CMA CGM, Hapag-Lloyd, ONE, OOCL]  \n2                        [Hapag-Lloyd, MSC, MSC]  \n3  [Hapag-Lloyd, MSC, Hapag-Lloyd, W.E.C. Lines]  \n4           [Hapag-Lloyd, MSC, Hapag-Lloyd, ONE]  ",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Lead time</th>\n      <th>Transfers</th>\n      <th>Origin</th>\n      <th>Destination</th>\n      <th>Departure</th>\n      <th>Arrival</th>\n      <th>kg CO2e/TEU</th>\n      <th>Ports</th>\n      <th>Carriers</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>11 days</td>\n      <td>1</td>\n      <td>BRFOR</td>\n      <td>NLRTM</td>\n      <td>[21 OCT,  02:57, UTC-03:00]</td>\n      <td>[1 NOV,  06:00, UTC+01:00]</td>\n      <td>700</td>\n      <td>[[FOR, PEC], [PEC, PTM, RTM]]</td>\n      <td>[Hapag-Lloyd, MSC]</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>23 days</td>\n      <td>1</td>\n      <td>BRFOR</td>\n      <td>BEANR</td>\n      <td>[27 OCT,  00:00, UTC-03:00]</td>\n      <td>[18 NOV,  22:00, UTC+01:00]</td>\n      <td>860</td>\n      <td>[[FOR, NAT, VGO, LGP, RTM], [RTM, BRV, ANR]]</td>\n      <td>[CMA CGM, CMA CGM, Hapag-Lloyd, ONE, OOCL]</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>16 days</td>\n      <td>2</td>\n      <td>BRFOR</td>\n      <td>BEANR</td>\n      <td>[21 OCT,  02:57, UTC-03:00]</td>\n      <td>[5 NOV,  22:00, UTC+01:00]</td>\n      <td>755</td>\n      <td>[[FOR, PEC], [PEC, PTM, RTM], [RTM, ANR]]</td>\n      <td>[Hapag-Lloyd, MSC, MSC]</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>22 days</td>\n      <td>2</td>\n      <td>BRFOR</td>\n      <td>BEANR</td>\n      <td>[21 OCT,  02:57, UTC-03:00]</td>\n      <td>[11 NOV,  18:00, UTC+01:00]</td>\n      <td>760</td>\n      <td>[[FOR, PEC], [PEC, PTM, RTM], [RTM, ANR]]</td>\n      <td>[Hapag-Lloyd, MSC, Hapag-Lloyd, W.E.C. Lines]</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>18 days</td>\n      <td>2</td>\n      <td>BRFOR</td>\n      <td>BEANR</td>\n      <td>[21 OCT,  02:57, UTC-03:00]</td>\n      <td>[8 NOV,  06:00, UTC+01:00]</td>\n      <td>780</td>\n      <td>[[FOR, PEC], [PEC, PTM, RTM, LGP], [LGP, ANR]]</td>\n      <td>[Hapag-Lloyd, MSC, Hapag-Lloyd, ONE]</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The data can be loaded from a pickle, so the script doesn't need to re-run every time.\n",
    "# Set use_pickl2 to False if you want to scrape all the data again.\n",
    "use_pickle2 = True\n",
    "\n",
    "if use_pickle2:\n",
    "    connection_df = pd.read_pickle(f\"../pickles/routescanner_daily/connections_{date}.pickle\")\n",
    "\n",
    "else:\n",
    "    # Empty list for data and len\n",
    "    connection_data = []\n",
    "\n",
    "    # Scrape the webpage\n",
    "    soups_ods_2 = get_webpages2(od_ports, headless=True)\n",
    "    # Get all the connection data, process them and add\n",
    "    connection_data = get_route_data2(soups_ods_2)\n",
    "\n",
    "    # Create a dataframe from all the route data\n",
    "    connection_df = pd.DataFrame(connection_data, columns=columns2)\n",
    "\n",
    "print(f\"DataFrame has {connection_df.index.size} entries\")\n",
    "connection_df.head()"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "# Save as Pickle and CSV\n",
    "connection_df.to_pickle(f\"../pickles/routescanner_daily/connections_{date}.pickle\")\n",
    "connection_df.to_csv(f\"../data/routescanner_daily/connections_{date}.csv\")"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}